fixed the tests

DEVELOPER.md

@@ -686,7 +686,7 @@ kubectl logs <vertica-pod-name> -c vlogger
          ...
          - args:
            - --health-probe-bind-address=:8081
-           - --metrics-bind-address=127.0.0.1:8443
+           - --metrics-bind-address=0.0.0.0:8443
            - --leader-elect
            - --health-probe-bind-address=:8081
            - --enable-profiler

Makefile

@@ -231,9 +231,35 @@ export VDB_MAX_BACKOFF_DURATION
 #
 # The address the operators Prometheus metrics endpoint binds to. Setting this
 # to 0 will disable metric serving.
-METRICS_ADDR?=127.0.0.1:8443
+METRICS_ADDR?=0.0.0.0:8443
 export METRICS_ADDR
 #
+# The secret name that will be used to mount cert files in the operator
+# for providing server certs to Prometheus metrics endpoint. Setting this
+# to "" will use an auto-generated self-signed cert.
+export METRICS_TLS_SECRET
+#
+# Controls exposing of the prometheus metrics endpoint. The valid values are:
+# EnableWithAuth: A new service object will be created that exposes the
+#    metrics endpoint.  Access to the metrics are controlled by rbac rules.
+#    The metrics endpoint will use the https scheme.
+# EnableWithoutAuth: Like EnableWithAuth, this will create a service
+#    object to expose the metrics endpoint.  However, there is no authority
+#    checking when using the endpoint.  Anyone who had network access
+#    endpoint (i.e. any pod in k8s) will be able to read the metrics.  The
+#    metrics endpoint will use the http scheme.
+# EnableWithTLS: Like EnableWithAuth, this will create a service
+#    object to expose the metrics endpoint.  However, there is no authority
+#    checking when using the endpoint.  People with network access to the
+#    endpoint (i.e. any pod in k8s) and the correct certs can read the metrics.
+#    The metrics endpoint will use the https scheme. 
+#    It needs to be used with tlsSecret. If tlsSecret is not set, the behavior
+#    will be similar to EnableWithoutAuth, except that the endpoint will use 
+#    https schema.
+# Disable: Prometheus metrics are not exposed at all.
+METRICS_EXPOSE_MODE?=Disable
+export METRICS_EXPOSE_MODE
+#
 # The minimum logging level. Valid values are: debug, info, warn, and error.
 LOG_LEVEL?=info
 export LOG_LEVEL

changes/unreleased/Added-20250131-221406.yaml

@@ -0,0 +1,6 @@
+kind: Added
+body: Added "EnableWithTLS" option to Helm parameter "prometheus.expose", allowing secure access
+  to metrics from outside the cluster
+time: 2025-01-31T22:14:06.675326382Z
+custom:
+  Issue: "1040"

changes/unreleased/Removed-20250131-221220.yaml

@@ -0,0 +1,5 @@
+kind: Removed
+body: Removed Helm parameter "prometheus.createServiceMonitor"
+time: 2025-01-31T22:12:20.085253713Z
+custom:
+  Issue: "1040"

cmd/operator/main.go

@@ -18,8 +18,10 @@ package main
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"log"
 	"os"
+	"strings"
 	"time"
 
 	// Allows us to pull in things generated from `go generate`
@@ -286,20 +288,42 @@ func main() {
 		TLSOpts: webhookTLSOpts,
 	})
 
-	secureMetrics := opcfg.GetMetricsAddr() == "127.0.0.1:8443"
+	secureMetrics := strings.EqualFold(opcfg.GetMetricsExposeMode(), "EnableWithAuth")
+	secureByTLS := strings.EqualFold(opcfg.GetMetricsExposeMode(), "EnableWithTLS")
 	var metricCertDir string
 	if opcfg.GetMetricsTLSSecret() != "" {
 		metricCertDir = "/cert"
+		metricsTLSOpts = append(metricsTLSOpts, func(c *tls.Config) {
+			// Load the CA certificate
+			caCert, err := os.ReadFile("/cert/ca.crt")
+			if err != nil {
+				log.Fatalf("failed to read CA cert: %v", err)
+			}
+			// Create a CertPool and add the CA certificate to it
+			caCertPool := x509.NewCertPool()
+			ok := caCertPool.AppendCertsFromPEM(caCert)
+			if !ok {
+				log.Fatal("failed to append CA cert to CertPool")
+			}
+			c.ClientCAs = caCertPool
+			// If we enabled authorization, then no client certs are really needed.
+			// Otherwise, we need the client certs.
+			if secureMetrics {
+				c.ClientAuth = tls.VerifyClientCertIfGiven
+			} else if secureByTLS {
+				c.ClientAuth = tls.RequireAndVerifyClientCert
+			}
+		})
 	}
 
 	// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
 	// More info:
 	// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/server
 	// - https://book.kubebuilder.io/reference/metrics.html
 	metricsServerOptions := metricsserver.Options{
-		BindAddress:   ":8443",
-		SecureServing: secureMetrics,
-		// TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are
+		BindAddress:   opcfg.GetMetricsAddr(),
+		SecureServing: secureMetrics || secureByTLS,
+		// TLSOpts is used to allow configuring the TLS config used for the server. If certificates are
 		// not provided, self-signed certificates will be generated by default. This option is not recommended for
 		// production environments as self-signed certificates do not offer the same level of trust and security
 		// as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing

config/default/kustomization.yaml

@@ -24,7 +24,7 @@ resources:
 - ../webhook
 - ../clusterpermissions
 - ../certmanager
-- ../prometheus
+# - ../prometheus
 - metrics_service.yaml
 
 # Protect the /metrics endpoint by putting it behind auth.

config/manager/operator-envs

@@ -7,7 +7,7 @@ CONTROLLERS_ENABLED=${CONTROLLERS_ENABLED}
 CONTROLLERS_SCOPE=${CONTROLLERS_SCOPE}
 METRICS_ADDR=${METRICS_ADDR}
 METRICS_TLS_SECRET=${METRICS_TLS_SECRET}
-METRICS_PROXY_RBAC=${METRICS_PROXY_RBAC}
+METRICS_EXPOSE_MODE=${METRICS_EXPOSE_MODE}
 LOG_LEVEL=${LOG_LEVEL}
 CONCURRENCY_VERTICADB=${CONCURRENCY_VERTICADB}
 CONCURRENCY_VERTICAAUTOSCALER=${CONCURRENCY_VERTICAAUTOSCALER}

config/rbac/kustomization.yaml

@@ -18,6 +18,7 @@ resources:
 - metrics_auth_role.yaml
 - metrics_auth_role_binding.yaml
 - metrics_reader_role.yaml
+- metrics_reader_role_binding.yaml
 # The next setup the RBAC rules for the webhook.
 - webhook_config_clusterrole.yaml
 - webhook_config_clusterrolebinding.yaml

config/rbac/metrics_reader_role_binding.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics-reader
+subjects:
+- kind: ServiceAccount
+  name: manager
+  namespace: system
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:authenticated
+

helm-charts/verticadb-operator/README.md

@@ -21,9 +21,8 @@ This helm chart will install the operator and an admission controller webhook.
 | nodeSelector | The [node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) provides control over which nodes are used to schedule a pod. If this parameter is not set, the node selector is omitted from the pod that is created by the operator's Deployment object. To set this parameter, provide a list of key/value pairs. | Not set |
 | priorityClassName | The [priority class name](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass) that is assigned to the operator pod. This affects where the pod gets scheduled. | Not set |
 | prometheus.createProxyRBAC | Set this to false if you want to avoid creating the rbac rules for accessing the metrics endpoint when it is protected by the rbac auth proxy.  By default, we will create those RBAC rules. | true |
-| prometheus.createServiceMonitor | Set this to true if you want to create a ServiceMonitor.  This object is a CR provided by the prometheus operator to allow for easy service discovery.  If set to true, the prometheus operator must be installed before installing this chart.<br> See: https://github.com/prometheus-operator/prometheus-operator<br><br>*This parameter is deprecated and will be removed in a future release.* | false |
-| prometheus.expose | Controls exposing of the prometheus metrics endpoint.  Valid options are:<br><br>- **EnableWithAuthProxy**: A new service object will be created that exposes the metrics endpoint.  Access to the metrics are controlled by rbac rules. The metrics endpoint will use the https scheme.<br><br>- **EnableWithoutAuth**: Like EnableWithAuthProxy, this will create a service object to expose the metrics endpoint.  However, there is no authority checking when using the endpoint.  Anyone who has network access to the endpoint (i.e. any pod in k8s) will be able to read the metrics.  The metrics endpoint will use the http scheme.<br><br>- **Disable**: Prometheus metrics are not exposed at all.  | Disable |
-| prometheus.tlsSecret | Use this if you want to provide your own certs for the prometheus metrics endpoint. It refers to a secret in the same namespace that the helm chart is deployed in.  The secret must have the following keys set:<br><br>- **tls.key** – private key<br>- **tls.crt** – cert for the private key<br>- **ca.crt** – CA certificate<br><br>The prometheus.expose=EnableWithAuthProxy must be set for the operator to use the certs provided. If this field is omitted, the operator will generate its own self-signed cert. | "" |
+| prometheus.expose | Controls exposing of the prometheus metrics endpoint.  Valid options are:<br><br>- **EnableWithAuth**: A new service object will be created that exposes the metrics endpoint.  Access to the metrics are controlled by rbac rules. The metrics endpoint will use the https scheme.<br><br>- **EnableWithoutAuth**: Like EnableWithAuth, this will create a service object to expose the metrics endpoint.  However, there is no authority checking when using the endpoint.  Anyone who has network access to the endpoint (i.e. any pod in k8s) will be able to read the metrics.  The metrics endpoint will use the http scheme.<br><br>- **EnableWithTLS**: Like EnableWithAuth, this will create a service object to expose the metrics endpoint. However, there is no authority checking when using the endpoint. People with network access to the endpoint (i.e., any pod in Kubernetes) and the correct certificates can read the metrics. The metrics endpoint will use HTTPS and must be used with `tlsSecret`. If `tlsSecret` is not set, the behavior will be similar to `EnableWithoutAuth`, except that the endpoint will use HTTPS.<br><br>- **Disable**: Prometheus metrics are not exposed at all.  | Disable |
+| prometheus.tlsSecret | Use this if you want to provide your own certs for the prometheus metrics endpoint. It refers to a secret in the same namespace that the helm chart is deployed in.  The secret must have the following keys set:<br><br>- **tls.key** – private key<br>- **tls.crt** – cert for the private key<br>- **ca.crt** – CA certificate<br><br>The prometheus.expose=EnableWithAuth must be set for the operator to use the certs provided. If this field is omitted, the operator will generate its own self-signed cert. | "" |
 | reconcileConcurrency.eventtrigger | Set this to control the concurrency of reconciliations of EventTrigger CRs | 1 |
 | reconcileConcurrency.sandboxconfigmap | Set this to control the concurrency of reconciliations of ConfigMaps that contain state for a sandbox  | 1 |
 | reconcileConcurrency.verticaautoscaler | Set this to control the concurrency of reconciliations of VerticaAutoscaler CRs | 1 |

helm-charts/verticadb-operator/tests/image-name-and-tag_test.yaml

@@ -21,7 +21,7 @@ tests:
       imagePullSecrets:
         - name: image-pull-secrets
       prometheus:
-        expose: EnableWithAuthProxy
+        expose: EnableWithAuth
     asserts:
       - equal:
           path: spec.template.spec.containers[0].image

helm-charts/verticadb-operator/tests/metrics-configmap_test.yaml

@@ -13,8 +13,8 @@ tests:
   - it: should cotain ip if expose is with auth
     set:
       prometheus:
-        expose: EnableWithAuthProxy
+        expose: EnableWithAuth
     asserts:
       - equal:
           path: data.METRICS_ADDR
-          value: 127.0.0.1:8443
+          value: 0.0.0.0:8443

helm-charts/verticadb-operator/tests/serviceaccount-rolebinding_test.yaml

@@ -2,13 +2,15 @@ suite: ServiceAccount tests
 templates:
   - verticadb-operator-manager-clusterrolebinding-crb.yaml
   - verticadb-operator-webhook-config-crb.yaml
+  - verticadb-operator-metrics-auth-rolebinding-crb.yaml
+  - verticadb-operator-metrics-reader-crb.yaml
   - verticadb-operator-leader-election-rolebinding-rb.yaml
 tests:
   - it: should include the serviceaccount name when an override is set
     set:
       serviceAccountNameOverride: special-override-sa
       prometheus:
-        expose: "EnableWithAuthProxy"
+        expose: "EnableWithAuth"
         createProxyRBAC: true
     asserts:
       - equal:

helm-charts/verticadb-operator/values.yaml

@@ -151,30 +151,32 @@ serviceAccountAnnotations: {}
 prometheus:
   # Controls exposing of the prometheus metrics endpoint.  Valid options are:
   #
-  # EnableWithAuthProxy: A new service object will be created that exposes the
+  # EnableWithAuth: A new service object will be created that exposes the
   #    metrics endpoint.  Access to the metrics are controlled by rbac rules.
   #    The metrics endpoint will use the https scheme.
-  # EnableWithoutAuth: Like EnableWithAuthProxy, this will create a service
+  # EnableWithoutAuth: Like EnableWithAuth, this will create a service
   #    object to expose the metrics endpoint.  However, there is no authority
   #    checking when using the endpoint.  Anyone who had network access
   #    endpoint (i.e. any pod in k8s) will be able to read the metrics.  The
   #    metrics endpoint will use the http scheme.
+  # EnableWithTLS: Like EnableWithAuth, this will create a service
+  #    object to expose the metrics endpoint.  However, there is no authority
+  #    checking when using the endpoint.  People with network access to the
+  #    endpoint (i.e. any pod in k8s) and the correct certs can read the metrics.
+  #    The metrics endpoint will use the https scheme. 
+  #    It needs to be used with tlsSecret. If tlsSecret is not set, the behavior
+  #    will be similar to EnableWithoutAuth, except that the endpoint will use 
+  #    https schema.
   # Disable: Prometheus metrics are not exposed at all.
   expose: Disable
 
-  # If prometheus is exposed with an auth proxy (EnableWithAuthProxy), use this
+  # If prometheus is exposed with an auth proxy (EnableWithAuth), use this
   # parameter to control what certificates are used for the https endpoint. If
   # this is empty, the operator will use a generated self-signed cert. When
   # provided, the certificates can be used to authenticate with the metrics
   # endpoint.
   tlsSecret: ""
 
-  # ** This parameter is deprecated and will be removed in a future release.
-  # Set this to true if you want to create a ServiceMonitor.  This object is a
-  # CR provided by the prometheus operator to allow for easy service discovery.
-  # https://github.com/prometheus-operator/prometheus-operator
-  createServiceMonitor: false
-
   # This controls the creation of ClusterRole/ClusterRoleBinding to access
   # the metrics endpoint.
   createProxyRBAC: true

pkg/opcfg/config.go

@@ -104,6 +104,11 @@ func GetMetricsTLSSecret() string {
 	return lookupStringEnvVar("METRICS_TLS_SECRET", envCanNotExist)
 }
 
+// GetMetricsExposeMode returns exposing mode of the manager's Prometheus endpoint.
+func GetMetricsExposeMode() string {
+	return lookupStringEnvVar("METRICS_EXPOSE_MODE", envCanNotExist)
+}
+
 // GetUseCertManager returns true if cert-manager is used to setup the webhook's
 // TLS certs.
 func GetUseCertManager() bool {

scripts/authorize-metrics.sh

@@ -63,6 +63,7 @@ set -o xtrace
 if [[ -n "$UNDO" ]]
 then
     kubectl delete -f $REPO_DIR/config/release-manifests/verticadb-operator-metrics-reader-cr.yaml || :
+    kubectl delete -f $REPO_DIR/config/release-manifests/verticadb-operator-metrics-reader-crb.yaml || :
     echo "Finished undoing action"
     exit 0
 fi

scripts/gen-csv.sh

@@ -95,11 +95,11 @@ perl -i -0777 -pe 's/\n\s*- mountPath: \/cert\s*\n\s*name: auth-cert//g' bundle/
 # requirement on having the Prometheus Operator installed.  We are only
 # optionally installing this.  We will include the manifest in our GitHub
 # artifacts and have it as an optional helm parameter.
-rm bundle/manifests/*servicemonitor.yaml
+rm -f bundle/manifests/*servicemonitor.yaml
 # Remove the metrics-reader clusterrolebinding. When undeploying olm installs,
 # the clusterrole would get removed but not the clusterrolebinding. We provide
 # this as an arifact anyway, so it doesn't need to be part of the bundle.
-rm bundle/manifests/*metrics-reader*yaml
+rm -f bundle/manifests/*metrics-reader*yaml
 
 # Add the supported versions at the end of annotations.yaml
 cat <<EOT >> bundle/metadata/annotations.yaml

scripts/gen-release-artifacts.sh

@@ -52,7 +52,7 @@ fi
 # command.
 RELEASE_ARTIFACT_TARGET_DIR=$REPO_DIR/config/release-manifests
 mkdir -p $RELEASE_ARTIFACT_TARGET_DIR
-for f in verticadb-operator-metrics-monitor-servicemonitor.yaml \
+for f in verticadb-operator-metrics-reader-crb.yaml \
     verticadb-operator-metrics-reader-cr.yaml
 do
   cp $MANIFEST_DIR/$f $RELEASE_ARTIFACT_TARGET_DIR