removing total executions running metric

core/services/workflows/engine.go

@@ -1153,7 +1153,6 @@ func (e *Engine) heartbeat(ctx context.Context) {
 			return
 		case <-ticker.C:
 			e.metrics.incrementEngineHeartbeatCounter(ctx)
-			e.metrics.updateTotalWorkflowsGauge(ctx, e.stepUpdatesChMap.len())
 			logCustMsg(ctx, e.cma, "engine heartbeat at: "+e.clock.Now().Format(time.RFC3339), e.logger)
 		}
 	}
@@ -1266,7 +1265,7 @@ const (
 	defaultQueueSize            = 100000
 	defaultNewWorkerTimeout     = 2 * time.Second
 	defaultMaxExecutionDuration = 10 * time.Minute
-	defaultHeartbeatCadence     = 5 * time.Minute
+	defaultHeartbeatCadence     = 1 * time.Minute
 	defaultStepTimeout          = 2 * time.Minute
 )
 

core/services/workflows/monitoring.go

@@ -16,12 +16,12 @@ import (
 // em AKA "engine metrics" is to locally scope these instruments to avoid
 // data races in testing
 type engineMetrics struct {
-	registerTriggerFailureCounter            metric.Int64Counter
-	triggerWorkflowStarterErrorCounter       metric.Int64Counter
-	workflowsRunningGauge                    metric.Int64Gauge
-	capabilityInvocationCounter              metric.Int64Counter
-	capabilityFailureCounter                 metric.Int64Counter
-	workflowRegisteredCounter                metric.Int64Counter
+	registerTriggerFailureCounter      metric.Int64Counter
+	triggerWorkflowStarterErrorCounter metric.Int64Counter
+	capabilityInvocationCounter        metric.Int64Counter
+	capabilityFailureCounter           metric.Int64Counter
+	workflowRegisteredCounter          metric.Int64Counter
+	// workflowUnregisteredCounter can also be viewed as workflow deletions
 	workflowUnregisteredCounter              metric.Int64Counter
 	workflowExecutionRateLimitGlobalCounter  metric.Int64Counter
 	workflowExecutionRateLimitPerUserCounter metric.Int64Counter
@@ -59,11 +59,6 @@ func initMonitoringResources() (em *engineMetrics, err error) {
 		return nil, fmt.Errorf("failed to register trigger workflow starter error counter: %w", err)
 	}
 
-	em.workflowsRunningGauge, err = beholder.GetMeter().Int64Gauge("platform_engine_workflow_count")
-	if err != nil {
-		return nil, fmt.Errorf("failed to register workflows running gauge: %w", err)
-	}
-
 	em.capabilityInvocationCounter, err = beholder.GetMeter().Int64Counter("platform_engine_capabilities_count")
 	if err != nil {
 		return nil, fmt.Errorf("failed to register capability invocation counter: %w", err)
@@ -226,11 +221,6 @@ func (c workflowsMetricLabeler) incrementTotalWorkflowStepErrorsCounter(ctx cont
 	c.em.workflowStepErrorCounter.Add(ctx, 1, metric.WithAttributes(otelLabels...))
 }
 
-func (c workflowsMetricLabeler) updateTotalWorkflowsGauge(ctx context.Context, val int64) {
-	otelLabels := monutils.KvMapToOtelAttributes(c.Labels)
-	c.em.workflowsRunningGauge.Record(ctx, val, metric.WithAttributes(otelLabels...))
-}
-
 func (c workflowsMetricLabeler) incrementEngineHeartbeatCounter(ctx context.Context) {
 	otelLabels := monutils.KvMapToOtelAttributes(c.Labels)
 	c.em.engineHeartbeatCounter.Add(ctx, 1, metric.WithAttributes(otelLabels...))