From d0e7b1bb2be5eae02a117d9125b73fd8761e4c1c Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 16 Jan 2024 17:57:36 -0800 Subject: [PATCH 1/2] Add metrics for mempool pending transaction size --- internal/mempool/mempool.go | 8 ++++++++ internal/mempool/metrics.gen.go | 7 +++++++ internal/mempool/metrics.go | 3 +++ 3 files changed, 18 insertions(+) diff --git a/internal/mempool/mempool.go b/internal/mempool/mempool.go index 084313cec..4061686a1 100644 --- a/internal/mempool/mempool.go +++ b/internal/mempool/mempool.go @@ -182,6 +182,11 @@ func (txmp *TxMempool) Size() int { return txSize + pendingSize } +// PendingSize returns the number of pending transactions in the mempool. +func (txmp *TxMempool) PendingSize() int { + return txmp.pendingTxs.Size() +} + // SizeBytes return the total sum in bytes of all the valid transactions in the // mempool. It is thread-safe. func (txmp *TxMempool) SizeBytes() int64 { @@ -519,6 +524,7 @@ func (txmp *TxMempool) Update( } txmp.metrics.Size.Set(float64(txmp.Size())) + txmp.metrics.PendingSize.Set(float64(txmp.PendingSize())) return nil } @@ -638,6 +644,7 @@ func (txmp *TxMempool) addNewTransaction(wtx *WrappedTx, res *abci.ResponseCheck txmp.metrics.TxSizeBytes.Observe(float64(wtx.Size())) txmp.metrics.Size.Set(float64(txmp.Size())) + txmp.metrics.PendingSize.Set(float64(txmp.PendingSize())) txmp.insertTx(wtx) txmp.logger.Debug( @@ -745,6 +752,7 @@ func (txmp *TxMempool) handleRecheckResult(tx types.Tx, res *abci.ResponseCheckT } txmp.metrics.Size.Set(float64(txmp.Size())) + txmp.metrics.PendingSize.Set(float64(txmp.PendingSize())) } // updateReCheckTxs updates the recheck cursors using the gossipIndex. For diff --git a/internal/mempool/metrics.gen.go b/internal/mempool/metrics.gen.go index 100c5e71c..9d6ec19f1 100644 --- a/internal/mempool/metrics.gen.go +++ b/internal/mempool/metrics.gen.go @@ -20,6 +20,12 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { Name: "size", Help: "Number of uncommitted transactions in the mempool.", }, labels).With(labelsAndValues...), + PendingSize: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "pending_size", + Help: "Number of pending transactions in the mempool.", + }, labels).With(labelsAndValues...), TxSizeBytes: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, @@ -58,6 +64,7 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { func NopMetrics() *Metrics { return &Metrics{ Size: discard.NewGauge(), + PendingSize: discard.NewGauge(), TxSizeBytes: discard.NewHistogram(), FailedTxs: discard.NewCounter(), RejectedTxs: discard.NewCounter(), diff --git a/internal/mempool/metrics.go b/internal/mempool/metrics.go index 532307635..494f102f9 100644 --- a/internal/mempool/metrics.go +++ b/internal/mempool/metrics.go @@ -18,6 +18,9 @@ type Metrics struct { // Number of uncommitted transactions in the mempool. Size metrics.Gauge + // Number of pending transactions in mempool + PendingSize metrics.Gauge + // Histogram of transaction sizes in bytes. TxSizeBytes metrics.Histogram `metrics_buckettype:"exp" metrics_bucketsizes:"1,3,7"` From 6ddf2079c146120996163e300c705a4518d07376 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 17 Jan 2024 09:36:03 -0800 Subject: [PATCH 2/2] Add expired tx count metrics --- internal/mempool/mempool.go | 1 + internal/mempool/mempool_test.go | 1 + internal/mempool/metrics.gen.go | 9 ++++++++- internal/mempool/metrics.go | 6 ++++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/internal/mempool/mempool.go b/internal/mempool/mempool.go index 4061686a1..b76e1f007 100644 --- a/internal/mempool/mempool.go +++ b/internal/mempool/mempool.go @@ -297,6 +297,7 @@ func (txmp *TxMempool) CheckTx( timestamp: time.Now().UTC(), height: txmp.height, expiredCallback: func(removeFromCache bool) { + txmp.metrics.ExpiredTxs.Add(1) if removeFromCache { txmp.cache.Remove(tx) } diff --git a/internal/mempool/mempool_test.go b/internal/mempool/mempool_test.go index 336357aa2..d90a12277 100644 --- a/internal/mempool/mempool_test.go +++ b/internal/mempool/mempool_test.go @@ -215,6 +215,7 @@ func TestTxMempool_Size(t *testing.T) { txmp := setup(t, client, 0) txs := checkTxs(ctx, t, txmp, 100, 0) require.Equal(t, len(txs), txmp.Size()) + require.Equal(t, 0, txmp.PendingSize()) require.Equal(t, int64(5690), txmp.SizeBytes()) rawTxs := make([]types.Tx, len(txs)) diff --git a/internal/mempool/metrics.gen.go b/internal/mempool/metrics.gen.go index 9d6ec19f1..c5a44f029 100644 --- a/internal/mempool/metrics.gen.go +++ b/internal/mempool/metrics.gen.go @@ -24,7 +24,7 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { Namespace: namespace, Subsystem: MetricsSubsystem, Name: "pending_size", - Help: "Number of pending transactions in the mempool.", + Help: "Number of pending transactions in mempool", }, labels).With(labelsAndValues...), TxSizeBytes: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ Namespace: namespace, @@ -52,6 +52,12 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { Name: "evicted_txs", Help: "Number of evicted transactions.", }, labels).With(labelsAndValues...), + ExpiredTxs: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "expired_txs", + Help: "Number of expired transactions.", + }, labels).With(labelsAndValues...), RecheckTimes: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, @@ -69,6 +75,7 @@ func NopMetrics() *Metrics { FailedTxs: discard.NewCounter(), RejectedTxs: discard.NewCounter(), EvictedTxs: discard.NewCounter(), + ExpiredTxs: discard.NewCounter(), RecheckTimes: discard.NewCounter(), } } diff --git a/internal/mempool/metrics.go b/internal/mempool/metrics.go index 494f102f9..eb296182d 100644 --- a/internal/mempool/metrics.go +++ b/internal/mempool/metrics.go @@ -41,6 +41,12 @@ type Metrics struct { //metrics:Number of evicted transactions. EvictedTxs metrics.Counter + // ExpiredTxs defines the number of expired transactions. These are valid + // transactions that passed CheckTx and existed in the mempool but were not + // get picked up in time and eventually got expired and removed from mempool + //metrics:Number of expired transactions. + ExpiredTxs metrics.Counter + // Number of times transactions are rechecked in the mempool. RecheckTimes metrics.Counter }