diff --git a/examples/openshift/main.jsonnet b/examples/openshift/main.jsonnet index 4f11990e..384f56aa 100644 --- a/examples/openshift/main.jsonnet +++ b/examples/openshift/main.jsonnet @@ -10,7 +10,7 @@ local kp = common+: { namespace: 'openshift-monitoring', versions+: { - pyrra: '0.7.0-rc.1', + pyrra: '0.7.0-rc.2', }, }, }, @@ -66,8 +66,7 @@ local kp = apiService+: { metadata+: { annotations+: { - // TODO: uncomment to enable TLS for the Pyrra API server. - // 'service.beta.openshift.io/serving-cert-secret-name': 'pyrra-api-tls', + 'service.beta.openshift.io/serving-cert-secret-name': 'pyrra-api-tls', }, }, }, @@ -84,9 +83,12 @@ local kp = c { args: [ 'api', - '--api-url=http://pyrra-kubernetes.openshift-monitoring.svc.cluster.local:9444', + '--api-url=https://pyrra-kubernetes.openshift-monitoring.svc.cluster.local:9444', '--prometheus-bearer-token-path=/var/run/secrets/tokens/pyrra-api', '--prometheus-url=https://thanos-querier.openshift-monitoring.svc.cluster.local:9091', + '--tls-cert-file=/etc/tls/private/tls.crt', + '--tls-private-key-file=/etc/tls/private/tls.key', + '--tls-client-ca-file=/etc/tls/certs/service-ca.crt', ], volumeMounts+: [{ name: 'pyrra-sa-token', @@ -94,13 +96,12 @@ local kp = readOnly: true, }, { name: 'trusted-ca', - mountPath: '/etc/ssl/certs', + mountPath: '/etc/tls/certs', + readOnly: true, + }, { + name: 'tls', + mountPath: '/etc/tls/private', readOnly: true, - // TODO: uncomment to enable TLS for the Pyrra API server. - // }, { - // name: 'tls', - // mountPath: '/etc/tls/private', - // readOnly: true, }], } for c in super.containers @@ -121,12 +122,47 @@ local kp = path: 'service-ca.crt', }], }, - // TODO: uncomment to enable TLS for the Pyrra API server. - // }, { - // name: 'tls', - // secret: { - // secretName: 'pyrra-api-tls', - // }, + }, { + name: 'tls', + secret: { + secretName: 'pyrra-api-tls', + }, + }], + }, + }, + }, + }, + + kubernetesService+: { + metadata+: { + annotations+: { + 'service.beta.openshift.io/serving-cert-secret-name': 'pyrra-kubernetes-tls', + }, + }, + }, + kubernetesDeployment+: { + spec+: { + template+: { + spec+: { + containers: [ + c { + args+: [ + '--tls-cert-file=/etc/tls/private/tls.crt', + '--tls-private-key-file=/etc/tls/private/tls.key', + ], + volumeMounts+: [{ + name: 'tls', + mountPath: '/etc/tls/private', + readOnly: true, + }], + } + for c in super.containers + ], + volumes+: [{ + name: 'tls', + secret: { + secretName: 'pyrra-kubernetes-tls', + }, }], }, }, diff --git a/examples/openshift/manifests/pyrra-apiDeployment.yaml b/examples/openshift/manifests/pyrra-apiDeployment.yaml index 0652af83..c9ebcea3 100644 --- a/examples/openshift/manifests/pyrra-apiDeployment.yaml +++ b/examples/openshift/manifests/pyrra-apiDeployment.yaml @@ -6,7 +6,7 @@ metadata: labels: app.kubernetes.io/component: api app.kubernetes.io/name: pyrra - app.kubernetes.io/version: 0.7.0-rc.1 + app.kubernetes.io/version: 0.7.0-rc.2 name: pyrra-api namespace: openshift-monitoring spec: @@ -24,15 +24,18 @@ spec: labels: app.kubernetes.io/component: api app.kubernetes.io/name: pyrra - app.kubernetes.io/version: 0.7.0-rc.1 + app.kubernetes.io/version: 0.7.0-rc.2 spec: containers: - args: - api - - --api-url=http://pyrra-kubernetes.openshift-monitoring.svc.cluster.local:9444 + - --api-url=https://pyrra-kubernetes.openshift-monitoring.svc.cluster.local:9444 - --prometheus-bearer-token-path=/var/run/secrets/tokens/pyrra-api - --prometheus-url=https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 - image: ghcr.io/pyrra-dev/pyrra:v0.7.0-rc.1 + - --tls-cert-file=/etc/tls/private/tls.crt + - --tls-private-key-file=/etc/tls/private/tls.key + - --tls-client-ca-file=/etc/tls/certs/service-ca.crt + image: ghcr.io/pyrra-dev/pyrra:v0.7.0-rc.2 name: pyrra ports: - containerPort: 9099 @@ -43,9 +46,12 @@ spec: - mountPath: /var/run/secrets/tokens name: pyrra-sa-token readOnly: true - - mountPath: /etc/ssl/certs + - mountPath: /etc/tls/certs name: trusted-ca readOnly: true + - mountPath: /etc/tls/private + name: tls + readOnly: true nodeSelector: kubernetes.io/os: linux serviceAccountName: pyrra-api @@ -61,3 +67,6 @@ spec: path: service-ca.crt name: openshift-service-ca.crt name: trusted-ca + - name: tls + secret: + secretName: pyrra-api-tls diff --git a/examples/openshift/manifests/pyrra-apiService.yaml b/examples/openshift/manifests/pyrra-apiService.yaml index 02b35714..abfc0ad5 100644 --- a/examples/openshift/manifests/pyrra-apiService.yaml +++ b/examples/openshift/manifests/pyrra-apiService.yaml @@ -1,11 +1,12 @@ apiVersion: v1 kind: Service metadata: - annotations: {} + annotations: + service.beta.openshift.io/serving-cert-secret-name: pyrra-api-tls labels: app.kubernetes.io/component: api app.kubernetes.io/name: pyrra - app.kubernetes.io/version: 0.7.0-rc.1 + app.kubernetes.io/version: 0.7.0-rc.2 name: pyrra-api namespace: openshift-monitoring spec: diff --git a/examples/openshift/manifests/pyrra-apiServiceAccount.yaml b/examples/openshift/manifests/pyrra-apiServiceAccount.yaml index f534bae4..bf837704 100644 --- a/examples/openshift/manifests/pyrra-apiServiceAccount.yaml +++ b/examples/openshift/manifests/pyrra-apiServiceAccount.yaml @@ -4,6 +4,6 @@ metadata: labels: app.kubernetes.io/component: api app.kubernetes.io/name: pyrra - app.kubernetes.io/version: 0.7.0-rc.1 + app.kubernetes.io/version: 0.7.0-rc.2 name: pyrra-api namespace: openshift-monitoring diff --git a/examples/openshift/manifests/pyrra-apiServiceMonitor.yaml b/examples/openshift/manifests/pyrra-apiServiceMonitor.yaml index 5d26d4c3..bd7e14f2 100644 --- a/examples/openshift/manifests/pyrra-apiServiceMonitor.yaml +++ b/examples/openshift/manifests/pyrra-apiServiceMonitor.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/component: api app.kubernetes.io/name: pyrra - app.kubernetes.io/version: 0.7.0-rc.1 + app.kubernetes.io/version: 0.7.0-rc.2 name: pyrra-api namespace: openshift-monitoring spec: diff --git a/examples/openshift/manifests/pyrra-kubernetesClusterRole.yaml b/examples/openshift/manifests/pyrra-kubernetesClusterRole.yaml index cd2766fc..88259c2f 100644 --- a/examples/openshift/manifests/pyrra-kubernetesClusterRole.yaml +++ b/examples/openshift/manifests/pyrra-kubernetesClusterRole.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/component: kubernetes app.kubernetes.io/name: pyrra - app.kubernetes.io/version: 0.7.0-rc.1 + app.kubernetes.io/version: 0.7.0-rc.2 name: pyrra-kubernetes namespace: openshift-monitoring rules: diff --git a/examples/openshift/manifests/pyrra-kubernetesClusterRoleBinding.yaml b/examples/openshift/manifests/pyrra-kubernetesClusterRoleBinding.yaml index 2d2701f0..0786b7f6 100644 --- a/examples/openshift/manifests/pyrra-kubernetesClusterRoleBinding.yaml +++ b/examples/openshift/manifests/pyrra-kubernetesClusterRoleBinding.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/component: kubernetes app.kubernetes.io/name: pyrra - app.kubernetes.io/version: 0.7.0-rc.1 + app.kubernetes.io/version: 0.7.0-rc.2 name: pyrra-kubernetes namespace: openshift-monitoring roleRef: diff --git a/examples/openshift/manifests/pyrra-kubernetesDeployment.yaml b/examples/openshift/manifests/pyrra-kubernetesDeployment.yaml index 771f2bf3..c6c52161 100644 --- a/examples/openshift/manifests/pyrra-kubernetesDeployment.yaml +++ b/examples/openshift/manifests/pyrra-kubernetesDeployment.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/component: kubernetes app.kubernetes.io/name: pyrra - app.kubernetes.io/version: 0.7.0-rc.1 + app.kubernetes.io/version: 0.7.0-rc.2 name: pyrra-kubernetes namespace: openshift-monitoring spec: @@ -22,18 +22,28 @@ spec: labels: app.kubernetes.io/component: kubernetes app.kubernetes.io/name: pyrra - app.kubernetes.io/version: 0.7.0-rc.1 + app.kubernetes.io/version: 0.7.0-rc.2 spec: containers: - args: - kubernetes - image: ghcr.io/pyrra-dev/pyrra:v0.7.0-rc.1 + - --tls-cert-file=/etc/tls/private/tls.crt + - --tls-private-key-file=/etc/tls/private/tls.key + image: ghcr.io/pyrra-dev/pyrra:v0.7.0-rc.2 name: pyrra ports: - containerPort: 9099 securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /etc/tls/private + name: tls + readOnly: true nodeSelector: kubernetes.io/os: linux serviceAccountName: pyrra-kubernetes + volumes: + - name: tls + secret: + secretName: pyrra-kubernetes-tls diff --git a/examples/openshift/manifests/pyrra-kubernetesService.yaml b/examples/openshift/manifests/pyrra-kubernetesService.yaml index a0f3b579..635cf0e4 100644 --- a/examples/openshift/manifests/pyrra-kubernetesService.yaml +++ b/examples/openshift/manifests/pyrra-kubernetesService.yaml @@ -1,10 +1,12 @@ apiVersion: v1 kind: Service metadata: + annotations: + service.beta.openshift.io/serving-cert-secret-name: pyrra-kubernetes-tls labels: app.kubernetes.io/component: kubernetes app.kubernetes.io/name: pyrra - app.kubernetes.io/version: 0.7.0-rc.1 + app.kubernetes.io/version: 0.7.0-rc.2 name: pyrra-kubernetes namespace: openshift-monitoring spec: diff --git a/examples/openshift/manifests/pyrra-kubernetesServiceAccount.yaml b/examples/openshift/manifests/pyrra-kubernetesServiceAccount.yaml index 7cd7a969..e9bbebd5 100644 --- a/examples/openshift/manifests/pyrra-kubernetesServiceAccount.yaml +++ b/examples/openshift/manifests/pyrra-kubernetesServiceAccount.yaml @@ -4,6 +4,6 @@ metadata: labels: app.kubernetes.io/component: kubernetes app.kubernetes.io/name: pyrra - app.kubernetes.io/version: 0.7.0-rc.1 + app.kubernetes.io/version: 0.7.0-rc.2 name: pyrra-kubernetes namespace: openshift-monitoring diff --git a/examples/openshift/manifests/pyrra-kubernetesServiceMonitor.yaml b/examples/openshift/manifests/pyrra-kubernetesServiceMonitor.yaml index 342c3ec0..4918905f 100644 --- a/examples/openshift/manifests/pyrra-kubernetesServiceMonitor.yaml +++ b/examples/openshift/manifests/pyrra-kubernetesServiceMonitor.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/component: kubernetes app.kubernetes.io/name: pyrra - app.kubernetes.io/version: 0.7.0-rc.1 + app.kubernetes.io/version: 0.7.0-rc.2 name: pyrra-kubernetes namespace: openshift-monitoring spec: diff --git a/kubernetes.go b/kubernetes.go index ecdd7f39..86835d44 100644 --- a/kubernetes.go +++ b/kubernetes.go @@ -58,7 +58,12 @@ func init() { // +kubebuilder:scaffold:scheme } -func cmdKubernetes(logger log.Logger, metricsAddr string, _, genericRules, disableWebhooks bool) int { +func cmdKubernetes( + logger log.Logger, + metricsAddr string, + _, genericRules, disableWebhooks bool, + certFile, privateKeyFile string, +) int { setupLog := ctrl.Log.WithName("setup") ctrl.SetLogger(zap.New(zap.UseDevMode(true))) @@ -127,6 +132,10 @@ func cmdKubernetes(logger log.Logger, metricsAddr string, _, genericRules, disab } gr.Add(func() error { + if certFile != "" && privateKeyFile != "" { + setupLog.Info("serving with TLS", "cert", certFile, "key", privateKeyFile) + return server.ListenAndServeTLS(certFile, privateKeyFile) + } return server.ListenAndServe() }, func(err error) { shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second) diff --git a/main.go b/main.go index 7d9b8470..69e5810a 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/tls" "embed" "fmt" "html/template" @@ -60,7 +61,9 @@ var CLI struct { PrometheusBearerTokenPath string `default:"" help:"Bearer token path"` PrometheusBasicAuthUsername string `default:"" help:"The HTTP basic authentication username"` PrometheusBasicAuthPassword promconfig.Secret `default:"" help:"The HTTP basic authentication password"` - PrometheusServiceCAPath string `default:"/etc/ssl/certs/service-ca.crt" help:"The path to the SA certificate to load"` + TLSCertFile string `default:"" help:"File containing the default x509 Certificate for HTTPS."` + TLSPrivateKeyFile string `default:"" help:"File containing the default x509 private key matching --tls-cert-file."` + TLSClientCAFile string `default:"" help:"File containing the CA certificate for the client"` } `cmd:"" help:"Runs Pyrra's API and UI."` Filesystem struct { ConfigFiles string `default:"/etc/pyrra/*.yaml" help:"The folder where Pyrra finds the config files to use. Any non yaml files will be ignored."` @@ -69,10 +72,12 @@ var CLI struct { GenericRules bool `default:"false" help:"Enabled generic recording rules generation to make it easier for tools like Grafana."` } `cmd:"" help:"Runs Pyrra's filesystem operator and backend for the API."` Kubernetes struct { - MetricsAddr string `default:":8080" help:"The address the metric endpoint binds to."` - ConfigMapMode bool `default:"false" help:"If the generated recording rules should instead be saved to config maps in the default Prometheus format."` - GenericRules bool `default:"false" help:"Enabled generic recording rules generation to make it easier for tools like Grafana."` - DisableWebhooks bool `default:"true" env:"DISABLE_WEBHOOKS" help:"Disable webhooks so the controller doesn't try to read certificates"` + MetricsAddr string `default:":8080" help:"The address the metric endpoint binds to."` + ConfigMapMode bool `default:"false" help:"If the generated recording rules should instead be saved to config maps in the default Prometheus format."` + GenericRules bool `default:"false" help:"Enabled generic recording rules generation to make it easier for tools like Grafana."` + DisableWebhooks bool `default:"true" env:"DISABLE_WEBHOOKS" help:"Disable webhooks so the controller doesn't try to read certificates"` + TLSCertFile string `default:"" help:"File containing the default x509 Certificate for HTTPS."` + TLSPrivateKeyFile string `default:"" help:"File containing the default x509 private key matching --tls-cert-file."` } `cmd:"" help:"Runs Pyrra's Kubernetes operator and backend for the API."` Generate struct { ConfigFiles string `default:"/etc/pyrra/*.yaml" help:"The folder where Pyrra finds the config files to use."` @@ -114,11 +119,11 @@ func main() { if CLI.API.PrometheusBearerTokenPath != "" { clientConfig.BearerTokenFile = CLI.API.PrometheusBearerTokenPath } - if CLI.API.PrometheusServiceCAPath != "" { - clientConfig.TLSConfig = promconfig.TLSConfig{CAFile: CLI.API.PrometheusServiceCAPath} + if CLI.API.TLSClientCAFile != "" { + clientConfig.TLSConfig = promconfig.TLSConfig{CAFile: CLI.API.TLSClientCAFile} } - roundTripper, err := promconfig.NewRoundTripperFromConfig(clientConfig, "pyrra") + roundTripper, err := promconfig.NewRoundTripperFromConfig(clientConfig, "prometheus") if err != nil { level.Error(logger).Log("msg", "failed to create API client round tripper", "err", err) os.Exit(1) @@ -151,6 +156,8 @@ func main() { CLI.API.APIURL, CLI.API.RoutePrefix, CLI.API.UIRoutePrefix, + CLI.API.TLSCertFile, + CLI.API.TLSPrivateKeyFile, ) case "filesystem": code = cmdFilesystem( @@ -168,6 +175,8 @@ func main() { CLI.Kubernetes.ConfigMapMode, CLI.Kubernetes.GenericRules, CLI.Kubernetes.DisableWebhooks, + CLI.Kubernetes.TLSCertFile, + CLI.Kubernetes.TLSPrivateKeyFile, ) case "generate": code = cmdGenerate( @@ -181,7 +190,14 @@ func main() { os.Exit(code) } -func cmdAPI(logger log.Logger, reg *prometheus.Registry, promClient api.Client, prometheusExternal, apiURL *url.URL, routePrefix, uiRoutePrefix string) int { +func cmdAPI( + logger log.Logger, + reg *prometheus.Registry, + promClient api.Client, + prometheusExternal, apiURL *url.URL, + routePrefix, uiRoutePrefix string, + tlsCertFile, tlsPrivateKeyFile string, +) int { build, err := fs.Sub(ui, "ui/build") if err != nil { level.Error(logger).Log("msg", "failed to read UI build files", "err", err) @@ -235,12 +251,28 @@ func cmdAPI(logger log.Logger, reg *prometheus.Registry, promClient api.Client, prometheusInterceptor := connectprometheus.NewInterceptor(reg) r.Route(routePrefix, func(r chi.Router) { + clientConfig := promconfig.HTTPClientConfig{ + TLSConfig: promconfig.TLSConfig{ + InsecureSkipVerify: true, + }, + } + + roundTripper, err := promconfig.NewRoundTripperFromConfig(clientConfig, "api") + if err != nil { + level.Error(logger).Log("msg", "failed to create API client round tripper", "err", err) + os.Exit(1) + } + + client := &http.Client{ + Transport: roundTripper, + } + objectiveService := &objectiveServer{ logger: log.WithPrefix(logger, "service", "objective"), promAPI: promAPI, client: newBackendClientCache( objectivesv1alpha1connect.NewObjectiveBackendServiceClient( - http.DefaultClient, + client, apiURL.String(), connect.WithInterceptors(prometheusInterceptor), ), @@ -321,11 +353,16 @@ func cmdAPI(logger log.Logger, reg *prometheus.Registry, promClient api.Client, { httpServer := &http.Server{ - Addr: ":9099", - Handler: h2c.NewHandler(r, &http2.Server{}), + Addr: ":9099", + Handler: h2c.NewHandler(r, &http2.Server{}), + TLSConfig: &tls.Config{}, } gr.Add( func() error { + if tlsCertFile != "" && tlsPrivateKeyFile != "" { + level.Info(logger).Log("msg", "serving using TLS", "cert", tlsCertFile, "key", tlsPrivateKeyFile) + return httpServer.ListenAndServeTLS(tlsCertFile, tlsPrivateKeyFile) + } return httpServer.ListenAndServe() }, func(error) {