Add check on latest_block_time

td2/rpc.go

@@ -51,7 +51,7 @@ func (cc *ChainConfig) newRpc() error {
 			l(msg)
 			return
 		}
-		if status.SyncInfo.CatchingUp {
+		if status.SyncInfo.CatchingUp || !isLatestBlockTimeCurrent(status.SyncInfo.LatestBlockTime, cc.Alerts.Stalled) {
 			msg = fmt.Sprint("🐢 node is not synced, skipping ", u)
 			syncing = true
 			down = true
@@ -162,7 +162,7 @@ func (cc *ChainConfig) monitorHealth(ctx context.Context, chainName string) {
 						alert("on the wrong network")
 						return
 					}
-					if status.SyncInfo.CatchingUp {
+					if status.SyncInfo.CatchingUp || !isLatestBlockTimeCurrent(status.SyncInfo.LatestBlockTime, cc.Alerts.Stalled) {
 						alert("not synced")
 						node.syncing = true
 						return
@@ -181,7 +181,6 @@ func (cc *ChainConfig) monitorHealth(ctx context.Context, chainName string) {
 					l(fmt.Sprintf("🟢 %-12s node %s is healthy", chainName, node.Url))
 				}(node)
 			}
-
 			if cc.client == nil {
 				e := cc.newRpc()
 				if e != nil {
@@ -238,3 +237,8 @@ func guessPublicEndpoint(u string) string {
 	}
 	return proto + matches[1] + port
 }
+
+// isLatestBlockTimeCurrent checks to see if the `latest_block_time` is within stalledMinutes minutes of UTC time.
+func isLatestBlockTimeCurrent(blockTime time.Time, stalledMinutes int) bool {
+	return time.Now().UTC().Sub(blockTime) < time.Minute*time.Duration(stalledMinutes)
+}

td2/tenderduty_test.go

@@ -0,0 +1,27 @@
+package tenderduty
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"time"
+)
+
+func TestIsLatestBlockTimeCurrent(t *testing.T) {
+	currentTime := time.Now().UTC()
+
+	// Check current time
+	blockTime := currentTime
+	assert.True(t, isLatestBlockTimeCurrent(blockTime, 10))
+
+	// Check time from two minutes ago
+	blockTime = currentTime.Add(-time.Minute * 2)
+	assert.True(t, isLatestBlockTimeCurrent(blockTime, 10))
+
+	// Check time from two hours ago
+	blockTime = currentTime.Add(-time.Hour * 2)
+	assert.False(t, isLatestBlockTimeCurrent(blockTime, 10))
+
+	// Check time two minutes in the future
+	blockTime = currentTime.Add(time.Minute * 2)
+	assert.True(t, isLatestBlockTimeCurrent(blockTime, 10))
+}