Update proxy features

entitlements/entitlements.go

@@ -16,6 +16,8 @@
 
 package entitlements
 
+import "github.com/gravitational/teleport/api/client/proto"
+
 type EntitlementKind string
 
 // The EntitlementKind list should be 1:1 with the Features & FeatureStrings in salescenter/product/product.go,
@@ -56,3 +58,67 @@ var AllEntitlements = []EntitlementKind{
 	ExternalAuditStorage, FeatureHiding, HSM, Identity, JoinActiveSessions, K8s, MobileDeviceManagement, OIDC, OktaSCIM,
 	OktaUserSync, Policy, SAML, SessionLocks, UpsellAlert, UsageReporting,
 }
+
+// BackfillFeatures ensures entitlements are backwards compatible
+// If Entitlements are present, there are no changes
+// If Entitlements are not present, sets the entitlements fields to legacy field values
+// TODO(michellescripts) remove in v18
+func BackfillFeatures(features *proto.Features) {
+	if len(features.Entitlements) > 0 {
+		return
+	}
+
+	features.Entitlements = getBaseEntitlements(features.GetEntitlements())
+
+	// Entitlements: All records are {enabled: false}; update to equal legacy feature value
+	features.Entitlements[string(ExternalAuditStorage)] = &proto.EntitlementInfo{Enabled: features.GetExternalAuditStorage()}
+	features.Entitlements[string(FeatureHiding)] = &proto.EntitlementInfo{Enabled: features.GetFeatureHiding()}
+	features.Entitlements[string(Identity)] = &proto.EntitlementInfo{Enabled: features.GetIdentityGovernance()}
+	features.Entitlements[string(JoinActiveSessions)] = &proto.EntitlementInfo{Enabled: features.GetJoinActiveSessions()}
+	features.Entitlements[string(MobileDeviceManagement)] = &proto.EntitlementInfo{Enabled: features.GetMobileDeviceManagement()}
+	features.Entitlements[string(OIDC)] = &proto.EntitlementInfo{Enabled: features.GetOIDC()}
+	features.Entitlements[string(Policy)] = &proto.EntitlementInfo{Enabled: features.GetPolicy().GetEnabled()}
+	features.Entitlements[string(SAML)] = &proto.EntitlementInfo{Enabled: features.GetSAML()}
+	features.Entitlements[string(K8s)] = &proto.EntitlementInfo{Enabled: features.GetKubernetes()}
+	features.Entitlements[string(App)] = &proto.EntitlementInfo{Enabled: features.GetApp()}
+	features.Entitlements[string(DB)] = &proto.EntitlementInfo{Enabled: features.GetDB()}
+	features.Entitlements[string(Desktop)] = &proto.EntitlementInfo{Enabled: features.GetDesktop()}
+	features.Entitlements[string(HSM)] = &proto.EntitlementInfo{Enabled: features.GetHSM()}
+
+	// set default Identity fields to legacy feature value
+	features.Entitlements[string(AccessLists)] = &proto.EntitlementInfo{Enabled: true, Limit: features.GetAccessList().GetCreateLimit()}
+	features.Entitlements[string(AccessMonitoring)] = &proto.EntitlementInfo{Enabled: features.GetAccessMonitoring().GetEnabled(), Limit: features.GetAccessMonitoring().GetMaxReportRangeLimit()}
+	features.Entitlements[string(AccessRequests)] = &proto.EntitlementInfo{Enabled: features.GetAccessRequests().MonthlyRequestLimit > 0, Limit: features.GetAccessRequests().GetMonthlyRequestLimit()}
+	features.Entitlements[string(DeviceTrust)] = &proto.EntitlementInfo{Enabled: features.GetDeviceTrust().GetEnabled(), Limit: features.GetDeviceTrust().GetDevicesUsageLimit()}
+	// override Identity Package features if Identity is enabled: set true and clear limit
+	if features.GetIdentityGovernance() {
+		features.Entitlements[string(AccessLists)] = &proto.EntitlementInfo{Enabled: true}
+		features.Entitlements[string(AccessMonitoring)] = &proto.EntitlementInfo{Enabled: true}
+		features.Entitlements[string(AccessRequests)] = &proto.EntitlementInfo{Enabled: true}
+		features.Entitlements[string(DeviceTrust)] = &proto.EntitlementInfo{Enabled: true}
+		features.Entitlements[string(OktaSCIM)] = &proto.EntitlementInfo{Enabled: true}
+		features.Entitlements[string(OktaUserSync)] = &proto.EntitlementInfo{Enabled: true}
+		features.Entitlements[string(SessionLocks)] = &proto.EntitlementInfo{Enabled: true}
+	}
+}
+
+// getBaseEntitlements takes a cloud entitlement set and returns a modules Entitlement set
+func getBaseEntitlements(protoEntitlements map[string]*proto.EntitlementInfo) map[string]*proto.EntitlementInfo {
+	all := AllEntitlements
+	result := make(map[string]*proto.EntitlementInfo, len(all))
+
+	for _, e := range all {
+		al, ok := protoEntitlements[string(e)]
+		if !ok {
+			result[string(e)] = &proto.EntitlementInfo{}
+			continue
+		}
+
+		result[string(e)] = &proto.EntitlementInfo{
+			Enabled: al.Enabled,
+			Limit:   al.Limit,
+		}
+	}
+
+	return result
+}

lib/service/connect.go

@@ -1135,7 +1135,7 @@ func (process *TeleportProcess) getConnector(clientIdentity, serverIdentity *sta
 	// Set cluster features and return successfully with a working connector.
 	// TODO(michellescripts) remove clone & compatibility check in v18
 	cloned := apiutils.CloneProtoMsg(pingResponse.GetServerFeatures())
-	supportEntitlementsCompatibility(cloned)
+	entitlements.BackfillFeatures(cloned)
 	process.setClusterFeatures(cloned)
 	process.setAuthSubjectiveAddr(pingResponse.RemoteAddr)
 	process.logger.InfoContext(process.ExitContext(), "features loaded from auth server", "identity", clientIdentity.ID.Role, "features", pingResponse.GetServerFeatures())
@@ -1144,70 +1144,6 @@ func (process *TeleportProcess) getConnector(clientIdentity, serverIdentity *sta
 	return newConn, nil
 }
 
-// supportEntitlementsCompatibility ensures entitlements are backwards compatible
-// If Entitlements are present, there are no changes
-// If Entitlements are not present, sets the entitlements fields to legacy field values
-// TODO(michellescripts) remove in v18
-func supportEntitlementsCompatibility(features *proto.Features) {
-	if len(features.Entitlements) > 0 {
-		return
-	}
-
-	features.Entitlements = getBaseEntitlements(features.GetEntitlements())
-
-	// Entitlements: All records are {enabled: false}; update to equal legacy feature value
-	features.Entitlements[string(entitlements.ExternalAuditStorage)] = &proto.EntitlementInfo{Enabled: features.GetExternalAuditStorage()}
-	features.Entitlements[string(entitlements.FeatureHiding)] = &proto.EntitlementInfo{Enabled: features.GetFeatureHiding()}
-	features.Entitlements[string(entitlements.Identity)] = &proto.EntitlementInfo{Enabled: features.GetIdentityGovernance()}
-	features.Entitlements[string(entitlements.JoinActiveSessions)] = &proto.EntitlementInfo{Enabled: features.GetJoinActiveSessions()}
-	features.Entitlements[string(entitlements.MobileDeviceManagement)] = &proto.EntitlementInfo{Enabled: features.GetMobileDeviceManagement()}
-	features.Entitlements[string(entitlements.OIDC)] = &proto.EntitlementInfo{Enabled: features.GetOIDC()}
-	features.Entitlements[string(entitlements.Policy)] = &proto.EntitlementInfo{Enabled: features.GetPolicy().GetEnabled()}
-	features.Entitlements[string(entitlements.SAML)] = &proto.EntitlementInfo{Enabled: features.GetSAML()}
-	features.Entitlements[string(entitlements.K8s)] = &proto.EntitlementInfo{Enabled: features.GetKubernetes()}
-	features.Entitlements[string(entitlements.App)] = &proto.EntitlementInfo{Enabled: features.GetApp()}
-	features.Entitlements[string(entitlements.DB)] = &proto.EntitlementInfo{Enabled: features.GetDB()}
-	features.Entitlements[string(entitlements.Desktop)] = &proto.EntitlementInfo{Enabled: features.GetDesktop()}
-	features.Entitlements[string(entitlements.HSM)] = &proto.EntitlementInfo{Enabled: features.GetHSM()}
-
-	// set default Identity fields to legacy feature value
-	features.Entitlements[string(entitlements.AccessLists)] = &proto.EntitlementInfo{Enabled: true, Limit: features.GetAccessList().GetCreateLimit()}
-	features.Entitlements[string(entitlements.AccessMonitoring)] = &proto.EntitlementInfo{Enabled: features.GetAccessMonitoring().GetEnabled(), Limit: features.GetAccessMonitoring().GetMaxReportRangeLimit()}
-	features.Entitlements[string(entitlements.AccessRequests)] = &proto.EntitlementInfo{Enabled: features.GetAccessRequests().MonthlyRequestLimit > 0, Limit: features.GetAccessRequests().GetMonthlyRequestLimit()}
-	features.Entitlements[string(entitlements.DeviceTrust)] = &proto.EntitlementInfo{Enabled: features.GetDeviceTrust().GetEnabled(), Limit: features.GetDeviceTrust().GetDevicesUsageLimit()}
-	// override Identity Package features if Identity is enabled: set true and clear limit
-	if features.GetIdentityGovernance() {
-		features.Entitlements[string(entitlements.AccessLists)] = &proto.EntitlementInfo{Enabled: true}
-		features.Entitlements[string(entitlements.AccessMonitoring)] = &proto.EntitlementInfo{Enabled: true}
-		features.Entitlements[string(entitlements.AccessRequests)] = &proto.EntitlementInfo{Enabled: true}
-		features.Entitlements[string(entitlements.DeviceTrust)] = &proto.EntitlementInfo{Enabled: true}
-		features.Entitlements[string(entitlements.OktaSCIM)] = &proto.EntitlementInfo{Enabled: true}
-		features.Entitlements[string(entitlements.OktaUserSync)] = &proto.EntitlementInfo{Enabled: true}
-		features.Entitlements[string(entitlements.SessionLocks)] = &proto.EntitlementInfo{Enabled: true}
-	}
-}
-
-// getBaseEntitlements takes a cloud entitlement set and returns a modules Entitlement set
-func getBaseEntitlements(protoEntitlements map[string]*proto.EntitlementInfo) map[string]*proto.EntitlementInfo {
-	all := entitlements.AllEntitlements
-	result := make(map[string]*proto.EntitlementInfo, len(all))
-
-	for _, e := range all {
-		al, ok := protoEntitlements[string(e)]
-		if !ok {
-			result[string(e)] = &proto.EntitlementInfo{}
-			continue
-		}
-
-		result[string(e)] = &proto.EntitlementInfo{
-			Enabled: al.Enabled,
-			Limit:   al.Limit,
-		}
-	}
-
-	return result
-}
-
 // newClient attempts to connect to either the proxy server or auth server
 // For config v3 and onwards, it will only connect to either the proxy (via tunnel) or the auth server (direct),
 // depending on what was specified in the config.

lib/service/connect_test.go

@@ -276,7 +276,7 @@ func Test_supportEntitlementsCompatibility(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			cloned := apiutils.CloneProtoMsg(tt.features)
 
-			supportEntitlementsCompatibility(cloned)
+			entitlements.BackfillFeatures(cloned)
 			require.Equal(t, tt.expected, cloned.Entitlements)
 		})
 	}

lib/service/service.go

@@ -4664,6 +4664,7 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error {
 			TracerProvider:            process.TracingProvider,
 			AutomaticUpgradesChannels: cfg.Proxy.AutomaticUpgradesChannels,
 			IntegrationAppHandler:     connectionsHandler,
+			FeatureWatchInterval:      utils.HalfJitter(web.DefaultFeatureWatchInterval * 2),
 		}
 		webHandler, err := web.NewHandler(webConfig)
 		if err != nil {

lib/web/apiserver.go

@@ -21,6 +21,7 @@
 package web
 
 import (
+	"cmp"
 	"compress/gzip"
 	"context"
 	"crypto/tls"
@@ -125,6 +126,9 @@ const (
 	IncludedResourceModeRequestable = "requestable"
 	// IncludedResourceModeAll describes that all resources, requestable and available, should be returned.
 	IncludedResourceModeAll = "all"
+	// DefaultFeatureWatchInterval is the default time in which the feature watcher
+	// should ping the auth server to check for updated features
+	DefaultFeatureWatchInterval = time.Minute * 5
 )
 
 // healthCheckAppServerFunc defines a function used to perform a health check
@@ -154,12 +158,8 @@ type Handler struct {
 	// userConns tracks amount of current active connections with user certificates.
 	userConns atomic.Int32
 
-	// ClusterFeatures contain flags for supported and unsupported features.
-	// Note: This field can become stale since it's only set on initial proxy
-	// startup. To get the latest feature flags you'll need to ping from the
-	// auth server.
-	// https://github.com/gravitational/teleport/issues/39161
-	ClusterFeatures proto.Features
+	// clusterFeatures contain flags for supported and unsupported features.
+	clusterFeatures proto.Features
 
 	// nodeWatcher is a services.NodeWatcher used by Assist to lookup nodes from
 	// the proxy's cache and get nodes in real time.
@@ -172,6 +172,13 @@ type Handler struct {
 	// an authenticated websocket so unauthenticated sockets dont get left
 	// open.
 	wsIODeadline time.Duration
+
+	// featureWatcherStop is a channel used to emit a stop signal to the
+	// features watcher goroutine
+	featureWatcherStop chan struct{}
+	// featureWatcherReady is a chan that the feature watcher closes
+	// to signal it is ready. Used in tests.
+	featureWatcherReady chan struct{}
 }
 
 // HandlerOption is a functional argument - an option that can be passed
@@ -315,6 +322,10 @@ type Config struct {
 
 	// IntegrationAppHandler handles App Access requests which use an Integration.
 	IntegrationAppHandler app.ServerHandler
+
+	// FeatureWatchInterval is the interval between pings to the auth server
+	// to fetch new cluster features
+	FeatureWatchInterval time.Duration
 }
 
 // SetDefaults ensures proper default values are set if
@@ -329,6 +340,8 @@ func (c *Config) SetDefaults() {
 	if c.PresenceChecker == nil {
 		c.PresenceChecker = client.RunPresenceTask
 	}
+
+	c.FeatureWatchInterval = cmp.Or(c.FeatureWatchInterval, DefaultFeatureWatchInterval)
 }
 
 type APIHandler struct {
@@ -385,10 +398,12 @@ func NewHandler(cfg Config, opts ...HandlerOption) (*APIHandler, error) {
 		log:                  newPackageLogger(),
 		logger:               slog.Default().With(teleport.ComponentKey, teleport.ComponentWeb),
 		clock:                clockwork.NewRealClock(),
-		ClusterFeatures:      cfg.ClusterFeatures,
+		clusterFeatures:      cfg.ClusterFeatures,
 		healthCheckAppServer: cfg.HealthCheckAppServer,
 		tracer:               cfg.TracerProvider.Tracer(teleport.ComponentWeb),
 		wsIODeadline:         wsIODeadline,
+		featureWatcherStop:   make(chan struct{}),
+		featureWatcherReady:  make(chan struct{}),
 	}
 
 	if automaticUpgrades(cfg.ClusterFeatures) && h.cfg.AutomaticUpgradesChannels == nil {
@@ -1110,17 +1125,12 @@ func (h *Handler) getUserContext(w http.ResponseWriter, r *http.Request, p httpr
 	}
 	desktopRecordingEnabled := recConfig.GetMode() != types.RecordOff
 
-	pingResp, err := clt.Ping(r.Context())
-	if err != nil {
-		return nil, trace.Wrap(err)
-	}
-
-	features := pingResp.GetServerFeatures()
-	entitlement := modules.GetProtoEntitlement(features, entitlements.AccessMonitoring)
+	features := h.GetClusterFeatures()
+	entitlement := modules.GetProtoEntitlement(&features, entitlements.AccessMonitoring)
 	// ensure entitlement is set & feature is configured
 	accessMonitoringEnabled := entitlement.Enabled && features.GetAccessMonitoringConfigured()
 
-	userContext, err := ui.NewUserContext(user, accessChecker.Roles(), *pingResp.ServerFeatures, desktopRecordingEnabled, accessMonitoringEnabled)
+	userContext, err := ui.NewUserContext(user, accessChecker.Roles(), features, desktopRecordingEnabled, accessMonitoringEnabled)
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -1626,14 +1636,7 @@ func (h *Handler) getWebConfig(w http.ResponseWriter, r *http.Request, p httprou
 		}
 	}
 
-	clusterFeatures := h.ClusterFeatures
-	// ping server to get cluster features since h.ClusterFeatures may be stale
-	pingResponse, err := h.GetProxyClient().Ping(r.Context())
-	if err != nil {
-		h.log.WithError(err).Warn("Cannot retrieve cluster features, client may receive stale features")
-	} else {
-		clusterFeatures = *pingResponse.ServerFeatures
-	}
+	clusterFeatures := h.GetClusterFeatures()
 
 	// get tunnel address to display on cloud instances
 	tunnelPublicAddr := ""
@@ -1747,7 +1750,6 @@ func setEntitlementsWithLegacyLogic(webCfg *webclient.WebConfig, clusterFeatures
 		webCfg.Entitlements[string(entitlements.OIDC)] = webclient.EntitlementInfo{Enabled: clusterFeatures.GetOIDC()}
 		webCfg.Entitlements[string(entitlements.Policy)] = webclient.EntitlementInfo{Enabled: clusterFeatures.GetPolicy() != nil && clusterFeatures.GetPolicy().Enabled}
 		webCfg.Entitlements[string(entitlements.SAML)] = webclient.EntitlementInfo{Enabled: clusterFeatures.GetSAML()}
-
 		// set default Identity fields to legacy feature value
 		webCfg.Entitlements[string(entitlements.AccessLists)] = webclient.EntitlementInfo{Enabled: true, Limit: clusterFeatures.GetAccessList().GetCreateLimit()}
 		webCfg.Entitlements[string(entitlements.AccessMonitoring)] = webclient.EntitlementInfo{Enabled: clusterFeatures.GetAccessMonitoring().GetEnabled(), Limit: clusterFeatures.GetAccessMonitoring().GetMaxReportRangeLimit()}