Add metrics to track vCenter session

go.mod

@@ -153,7 +153,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/prometheus/client_golang v1.17.0 // indirect
+	github.com/prometheus/client_golang v1.17.0
 	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect

pkg/session/session.go

@@ -23,12 +23,14 @@ import (
 	"fmt"
 	"net/netip"
 	"net/url"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/blang/semver"
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/vmware/govmomi"
 	"github.com/vmware/govmomi/find"
 	"github.com/vmware/govmomi/object"
@@ -40,18 +42,60 @@ import (
 	"github.com/vmware/govmomi/vim25/methods"
 	"github.com/vmware/govmomi/vim25/soap"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/constants"
 )
 
+const (
+	metricNameSpace        = "session"
+	metricLabelServer      = "server"
+	metricLabelDC          = "dc"
+	metricLabelUsername    = "username"
+	metricLabelSessionKey  = "sessionKey"
+	metricLabelSessionKeys = "sessionKeys"
+)
+
 var (
 	// global Session map against sessionKeys in map[sessionKey]Session.
 	sessionCache sync.Map
 
 	// mutex to control access to the GetOrCreate function to avoid duplicate
 	// session creations on startup.
 	sessionMU sync.Mutex
+
+	sessionCacheMetric = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: metricNameSpace,
+			Name:      "cached_num",
+		},
+		[]string{metricLabelSessionKeys},
+	)
+
+	getSessionMetric = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricNameSpace,
+			Name:      "get",
+		},
+		[]string{metricLabelServer, metricLabelDC, metricLabelUsername},
+	)
+
+	createSessionMetric = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricNameSpace,
+			Name:      "create",
+		},
+		[]string{metricLabelServer, metricLabelDC, metricLabelUsername},
+	)
+
+	deleteSessionMetric = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricNameSpace,
+			Name:      "delete",
+		},
+		[]string{metricLabelSessionKey},
+	)
 )
 
 // Session is a vSphere session with a configured Finder.
@@ -84,6 +128,27 @@ type Params struct {
 	feature    Feature
 }
 
+func init() {
+	metrics.Registry.MustRegister(sessionCacheMetric, getSessionMetric, createSessionMetric, deleteSessionMetric)
+	ticker := time.NewTicker(1 * time.Minute)
+
+	go func() {
+		for range ticker.C {
+			size := 0
+			var sessionKeys []string
+			sessionCache.Range(func(key, value interface{}) bool {
+				size++
+				sessionKeys = append(sessionKeys, key.(string))
+				return true
+			})
+			sessionKeyLabelVal := strings.Join(sessionKeys, ",")
+			sessionCacheMetric.With(prometheus.Labels{
+				metricLabelSessionKeys: sessionKeyLabelVal,
+			}).Set(float64(size))
+		}
+	}()
+}
+
 // NewParams returns an empty set of parameters with default features.
 func NewParams() *Params {
 	return &Params{
@@ -139,6 +204,11 @@ func GetOrCreate(ctx context.Context, params *Params) (*Session, error) {
 	hashedUserPassword := h.Sum(nil)
 	sessionKey := fmt.Sprintf("%s#%s#%s#%x", params.server, params.datacenter, params.userinfo.Username(),
 		hashedUserPassword)
+	getSessionMetric.With(prometheus.Labels{
+		metricLabelServer:   params.server,
+		metricLabelDC:       params.datacenter,
+		metricLabelUsername: params.userinfo.Username(),
+	}).Inc()
 	if cachedSession, ok := sessionCache.Load(sessionKey); ok {
 		s := cachedSession.(*Session)
 
@@ -167,6 +237,12 @@ func GetOrCreate(ctx context.Context, params *Params) (*Session, error) {
 		}
 	}
 
+	createSessionMetric.With(prometheus.Labels{
+		metricLabelServer:   params.server,
+		metricLabelDC:       params.datacenter,
+		metricLabelUsername: params.userinfo.Username(),
+	}).Inc()
+
 	// soap.ParseURL expects a valid URL. In the case of a bare, unbracketed
 	// IPv6 address (e.g fd00::1) ParseURL will fail. Surround unbracketed IPv6
 	// addresses with brackets.
@@ -243,6 +319,9 @@ func newClient(ctx context.Context, logger logr.Logger, sessionKey string, url *
 			if err != nil {
 				logger.Error(err, "failed to keep alive govmomi client")
 				logger.Info("clearing the session")
+				deleteSessionMetric.With(prometheus.Labels{
+					metricLabelSessionKey: sessionKey,
+				}).Inc()
 				sessionCache.Delete(sessionKey)
 			}
 			return err
@@ -270,6 +349,9 @@ func newManager(ctx context.Context, logger logr.Logger, sessionKey string, clie
 			}
 
 			logger.Info("rest client session expired, clearing session")
+			deleteSessionMetric.With(prometheus.Labels{
+				metricLabelSessionKey: sessionKey,
+			}).Inc()
 			sessionCache.Delete(sessionKey)
 			return errors.New("rest client session expired")
 		})