add drained nonce cache metrics

strangelove-ventures · Nov 17, 2023 · 1cb86af · 1cb86af
1 parent cef385c
commit 1cb86af
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 1 deletion.
diff --git a/signer/metrics.go b/signer/metrics.go
@@ -214,6 +214,18 @@ var (
 		},
 		[]string{"peerid"},
 	)
+	drainedNonceCache = promauto.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "signer_drained_nonce_cache",
+			Help: "Consecutive Nonces Requested When Cache is Drained",
+		},
+	)
+	totalDrainedNonceCache = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "signer_total_drained_nonce_cache",
+			Help: "Total Nonces Requested When Cache is Drained",
+		},
+	)
 
 	sentryConnectTries = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{

diff --git a/signer/threshold_validator.go b/signer/threshold_validator.go
@@ -465,6 +465,9 @@ func (pv *ThresholdValidator) getNoncesFallback(
 ) (*CosignerUUIDNonces, []Cosigner, error) {
 	nonces := make(map[Cosigner]CosignerNonces)
 
+	drainedNonceCache.Inc()
+	totalDrainedNonceCache.Inc()
+
 	var wg sync.WaitGroup
 	wg.Add(pv.threshold)
 
@@ -527,7 +530,7 @@ func (pv *ThresholdValidator) waitForPeerNonces(
 	peerStartTime := time.Now()
 	peerNonces, err := peer.GetNonces(ctx, []uuid.UUID{u})
 	if err != nil {
-		missedNonces.WithLabelValues(peer.GetAddress()).Add(float64(1))
+		missedNonces.WithLabelValues(peer.GetAddress()).Inc()
 		totalMissedNonces.WithLabelValues(peer.GetAddress()).Inc()
 
 		pv.logger.Error("Error getting nonces", "cosigner", peer.GetID(), "err", err)
@@ -668,6 +671,8 @@ func (pv *ThresholdValidator) Sign(ctx context.Context, chainID string, block Bl
 			return nil, stamp, fmt.Errorf("failed to get nonces: %w", errors.Join(err, fallbackErr))
 		}
 		dontIterateFastestCosigners = true
+	} else {
+		drainedNonceCache.Set(0)
 	}
 
 	nextFastestCosignerIndex := pv.threshold - 1