examples/openshift: Deploy Pyrra with mTLS

This encrypts the communication on OpenShift between the API and Kubernetes backend.
pyrra-dev · Sep 23, 2023 · e02a50c · e02a50c
1 parent 9b40aa8
commit e02a50c
Show file tree

Hide file tree

Showing 13 changed files with 150 additions and 46 deletions.
diff --git a/examples/openshift/main.jsonnet b/examples/openshift/main.jsonnet
@@ -10,7 +10,7 @@ local kp =
       common+: {
         namespace: 'openshift-monitoring',
         versions+: {
-          pyrra: '0.7.0-rc.1',
+          pyrra: '0.7.0-rc.2',
         },
       },
     },
@@ -66,8 +66,7 @@ local kp =
       apiService+: {
         metadata+: {
           annotations+: {
-            // TODO: uncomment to enable TLS for the Pyrra API server.
-            // 'service.beta.openshift.io/serving-cert-secret-name': 'pyrra-api-tls',
+            'service.beta.openshift.io/serving-cert-secret-name': 'pyrra-api-tls',
           },
         },
       },
@@ -84,23 +83,25 @@ local kp =
                 c {
                   args: [
                     'api',
-                    '--api-url=http://pyrra-kubernetes.openshift-monitoring.svc.cluster.local:9444',
+                    '--api-url=https://pyrra-kubernetes.openshift-monitoring.svc.cluster.local:9444',
                     '--prometheus-bearer-token-path=/var/run/secrets/tokens/pyrra-api',
                     '--prometheus-url=https://thanos-querier.openshift-monitoring.svc.cluster.local:9091',
+                    '--tls-cert-file=/etc/tls/private/tls.crt',
+                    '--tls-private-key-file=/etc/tls/private/tls.key',
+                    '--tls-client-ca-file=/etc/tls/certs/service-ca.crt',
                   ],
                   volumeMounts+: [{
                     name: 'pyrra-sa-token',
                     mountPath: '/var/run/secrets/tokens',
                     readOnly: true,
                   }, {
                     name: 'trusted-ca',
-                    mountPath: '/etc/ssl/certs',
+                    mountPath: '/etc/tls/certs',
+                    readOnly: true,
+                  }, {
+                    name: 'tls',
+                    mountPath: '/etc/tls/private',
                     readOnly: true,
-                    // TODO: uncomment to enable TLS for the Pyrra API server.
-                    // }, {
-                    //   name: 'tls',
-                    //   mountPath: '/etc/tls/private',
-                    //   readOnly: true,
                   }],
                 }
                 for c in super.containers
@@ -121,12 +122,47 @@ local kp =
                     path: 'service-ca.crt',
                   }],
                 },
-                // TODO: uncomment to enable TLS for the Pyrra API server.
-                // }, {
-                //   name: 'tls',
-                //   secret: {
-                //     secretName: 'pyrra-api-tls',
-                //   },
+              }, {
+                name: 'tls',
+                secret: {
+                  secretName: 'pyrra-api-tls',
+                },
+              }],
+            },
+          },
+        },
+      },
+
+      kubernetesService+: {
+        metadata+: {
+          annotations+: {
+            'service.beta.openshift.io/serving-cert-secret-name': 'pyrra-kubernetes-tls',
+          },
+        },
+      },
+      kubernetesDeployment+: {
+        spec+: {
+          template+: {
+            spec+: {
+              containers: [
+                c {
+                  args+: [
+                    '--tls-cert-file=/etc/tls/private/tls.crt',
+                    '--tls-private-key-file=/etc/tls/private/tls.key',
+                  ],
+                  volumeMounts+: [{
+                    name: 'tls',
+                    mountPath: '/etc/tls/private',
+                    readOnly: true,
+                  }],
+                }
+                for c in super.containers
+              ],
+              volumes+: [{
+                name: 'tls',
+                secret: {
+                  secretName: 'pyrra-kubernetes-tls',
+                },
               }],
             },
           },

diff --git a/examples/openshift/manifests/pyrra-apiDeployment.yaml b/examples/openshift/manifests/pyrra-apiDeployment.yaml
@@ -6,7 +6,7 @@ metadata:
   labels:
     app.kubernetes.io/component: api
     app.kubernetes.io/name: pyrra
-    app.kubernetes.io/version: 0.7.0-rc.1
+    app.kubernetes.io/version: 0.7.0-rc.2
   name: pyrra-api
   namespace: openshift-monitoring
 spec:
@@ -24,15 +24,18 @@ spec:
       labels:
         app.kubernetes.io/component: api
         app.kubernetes.io/name: pyrra
-        app.kubernetes.io/version: 0.7.0-rc.1
+        app.kubernetes.io/version: 0.7.0-rc.2
     spec:
       containers:
       - args:
         - api
-        - --api-url=http://pyrra-kubernetes.openshift-monitoring.svc.cluster.local:9444
+        - --api-url=https://pyrra-kubernetes.openshift-monitoring.svc.cluster.local:9444
         - --prometheus-bearer-token-path=/var/run/secrets/tokens/pyrra-api
         - --prometheus-url=https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
-        image: ghcr.io/pyrra-dev/pyrra:v0.7.0-rc.1
+        - --tls-cert-file=/etc/tls/private/tls.crt
+        - --tls-private-key-file=/etc/tls/private/tls.key
+        - --tls-client-ca-file=/etc/tls/certs/service-ca.crt
+        image: ghcr.io/pyrra-dev/pyrra:v0.7.0-rc.2
         name: pyrra
         ports:
         - containerPort: 9099
@@ -43,9 +46,12 @@ spec:
         - mountPath: /var/run/secrets/tokens
           name: pyrra-sa-token
           readOnly: true
-        - mountPath: /etc/ssl/certs
+        - mountPath: /etc/tls/certs
           name: trusted-ca
           readOnly: true
+        - mountPath: /etc/tls/private
+          name: tls
+          readOnly: true
       nodeSelector:
         kubernetes.io/os: linux
       serviceAccountName: pyrra-api
@@ -61,3 +67,6 @@ spec:
             path: service-ca.crt
           name: openshift-service-ca.crt
         name: trusted-ca
+      - name: tls
+        secret:
+          secretName: pyrra-api-tls
diff --git a/examples/openshift/manifests/pyrra-apiService.yaml b/examples/openshift/manifests/pyrra-apiService.yaml
@@ -1,11 +1,12 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations: {}
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: pyrra-api-tls
   labels:
     app.kubernetes.io/component: api
     app.kubernetes.io/name: pyrra
-    app.kubernetes.io/version: 0.7.0-rc.1
+    app.kubernetes.io/version: 0.7.0-rc.2
   name: pyrra-api
   namespace: openshift-monitoring
 spec:

diff --git a/examples/openshift/manifests/pyrra-apiServiceAccount.yaml b/examples/openshift/manifests/pyrra-apiServiceAccount.yaml
@@ -4,6 +4,6 @@ metadata:
   labels:
     app.kubernetes.io/component: api
     app.kubernetes.io/name: pyrra
-    app.kubernetes.io/version: 0.7.0-rc.1
+    app.kubernetes.io/version: 0.7.0-rc.2
   name: pyrra-api
   namespace: openshift-monitoring
diff --git a/examples/openshift/manifests/pyrra-apiServiceMonitor.yaml b/examples/openshift/manifests/pyrra-apiServiceMonitor.yaml
@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: api
     app.kubernetes.io/name: pyrra
-    app.kubernetes.io/version: 0.7.0-rc.1
+    app.kubernetes.io/version: 0.7.0-rc.2
   name: pyrra-api
   namespace: openshift-monitoring
 spec:

diff --git a/examples/openshift/manifests/pyrra-kubernetesClusterRole.yaml b/examples/openshift/manifests/pyrra-kubernetesClusterRole.yaml
@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: kubernetes
     app.kubernetes.io/name: pyrra
-    app.kubernetes.io/version: 0.7.0-rc.1
+    app.kubernetes.io/version: 0.7.0-rc.2
   name: pyrra-kubernetes
   namespace: openshift-monitoring
 rules:

diff --git a/examples/openshift/manifests/pyrra-kubernetesClusterRoleBinding.yaml b/examples/openshift/manifests/pyrra-kubernetesClusterRoleBinding.yaml
@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: kubernetes
     app.kubernetes.io/name: pyrra
-    app.kubernetes.io/version: 0.7.0-rc.1
+    app.kubernetes.io/version: 0.7.0-rc.2
   name: pyrra-kubernetes
   namespace: openshift-monitoring
 roleRef:

diff --git a/examples/openshift/manifests/pyrra-kubernetesDeployment.yaml b/examples/openshift/manifests/pyrra-kubernetesDeployment.yaml
@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: kubernetes
     app.kubernetes.io/name: pyrra
-    app.kubernetes.io/version: 0.7.0-rc.1
+    app.kubernetes.io/version: 0.7.0-rc.2
   name: pyrra-kubernetes
   namespace: openshift-monitoring
 spec:
@@ -22,18 +22,28 @@ spec:
       labels:
         app.kubernetes.io/component: kubernetes
         app.kubernetes.io/name: pyrra
-        app.kubernetes.io/version: 0.7.0-rc.1
+        app.kubernetes.io/version: 0.7.0-rc.2
     spec:
       containers:
       - args:
         - kubernetes
-        image: ghcr.io/pyrra-dev/pyrra:v0.7.0-rc.1
+        - --tls-cert-file=/etc/tls/private/tls.crt
+        - --tls-private-key-file=/etc/tls/private/tls.key
+        image: ghcr.io/pyrra-dev/pyrra:v0.7.0-rc.2
         name: pyrra
         ports:
         - containerPort: 9099
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /etc/tls/private
+          name: tls
+          readOnly: true
       nodeSelector:
         kubernetes.io/os: linux
       serviceAccountName: pyrra-kubernetes
+      volumes:
+      - name: tls
+        secret:
+          secretName: pyrra-kubernetes-tls
diff --git a/examples/openshift/manifests/pyrra-kubernetesService.yaml b/examples/openshift/manifests/pyrra-kubernetesService.yaml
@@ -1,10 +1,12 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: pyrra-kubernetes-tls
   labels:
     app.kubernetes.io/component: kubernetes
     app.kubernetes.io/name: pyrra
-    app.kubernetes.io/version: 0.7.0-rc.1
+    app.kubernetes.io/version: 0.7.0-rc.2
   name: pyrra-kubernetes
   namespace: openshift-monitoring
 spec:

diff --git a/examples/openshift/manifests/pyrra-kubernetesServiceAccount.yaml b/examples/openshift/manifests/pyrra-kubernetesServiceAccount.yaml
@@ -4,6 +4,6 @@ metadata:
   labels:
     app.kubernetes.io/component: kubernetes
     app.kubernetes.io/name: pyrra
-    app.kubernetes.io/version: 0.7.0-rc.1
+    app.kubernetes.io/version: 0.7.0-rc.2
   name: pyrra-kubernetes
   namespace: openshift-monitoring
diff --git a/examples/openshift/manifests/pyrra-kubernetesServiceMonitor.yaml b/examples/openshift/manifests/pyrra-kubernetesServiceMonitor.yaml
@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: kubernetes
     app.kubernetes.io/name: pyrra
-    app.kubernetes.io/version: 0.7.0-rc.1
+    app.kubernetes.io/version: 0.7.0-rc.2
   name: pyrra-kubernetes
   namespace: openshift-monitoring
 spec:

diff --git a/kubernetes.go b/kubernetes.go
@@ -58,7 +58,12 @@ func init() {
 	// +kubebuilder:scaffold:scheme
 }
 
-func cmdKubernetes(logger log.Logger, metricsAddr string, _, genericRules, disableWebhooks bool) int {
+func cmdKubernetes(
+	logger log.Logger,
+	metricsAddr string,
+	_, genericRules, disableWebhooks bool,
+	certFile, privateKeyFile string,
+) int {
 	setupLog := ctrl.Log.WithName("setup")
 	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
 
@@ -127,6 +132,10 @@ func cmdKubernetes(logger log.Logger, metricsAddr string, _, genericRules, disab
 		}
 
 		gr.Add(func() error {
+			if certFile != "" && privateKeyFile != "" {
+				setupLog.Info("serving with TLS", "cert", certFile, "key", privateKeyFile)
+				return server.ListenAndServeTLS(certFile, privateKeyFile)
+			}
 			return server.ListenAndServe()
 		}, func(err error) {
 			shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)