Add check on latest_block_time

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/go-yaml/yaml v2.1.0+incompatible
 	github.com/gorilla/websocket v1.5.0
 	github.com/prometheus/client_golang v1.12.2
+	github.com/stretchr/testify v1.8.0
 	github.com/tendermint/tendermint v0.34.24
 	github.com/textileio/go-threads v1.1.5
 	golang.org/x/crypto v0.1.0
@@ -90,7 +91,6 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.13.0 // indirect
-	github.com/stretchr/testify v1.8.0 // indirect
 	github.com/subosito/gotenv v1.4.1 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
 	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect

td2/rpc.go

@@ -51,7 +51,7 @@ func (cc *ChainConfig) newRpc() error {
 			l(msg)
 			return
 		}
-		if status.SyncInfo.CatchingUp {
+		if status.SyncInfo.CatchingUp || !isLatestBlockTimeCurrent(status.SyncInfo.LatestBlockTime, cc.Alerts.Stalled) {
 			msg = fmt.Sprint("🐢 node is not synced, skipping ", u)
 			syncing = true
 			down = true
@@ -162,7 +162,7 @@ func (cc *ChainConfig) monitorHealth(ctx context.Context, chainName string) {
 						alert("on the wrong network")
 						return
 					}
-					if status.SyncInfo.CatchingUp {
+					if status.SyncInfo.CatchingUp || !isLatestBlockTimeCurrent(status.SyncInfo.LatestBlockTime, cc.Alerts.Stalled) {
 						alert("not synced")
 						node.syncing = true
 						return
@@ -181,7 +181,6 @@ func (cc *ChainConfig) monitorHealth(ctx context.Context, chainName string) {
 					l(fmt.Sprintf("🟢 %-12s node %s is healthy", chainName, node.Url))
 				}(node)
 			}
-
 			if cc.client == nil {
 				e := cc.newRpc()
 				if e != nil {
@@ -238,3 +237,8 @@ func guessPublicEndpoint(u string) string {
 	}
 	return proto + matches[1] + port
 }
+
+// isLatestBlockTimeCurrent checks to see if the `latest_block_time` is within stalledMinutes minutes of UTC time.
+func isLatestBlockTimeCurrent(blockTime time.Time, stalledMinutes int) bool {
+	return time.Now().UTC().Sub(blockTime) < time.Minute*time.Duration(stalledMinutes)
+}

td2/tenderduty_test.go

@@ -0,0 +1,27 @@
+package tenderduty
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"time"
+)
+
+func TestIsLatestBlockTimeCurrent(t *testing.T) {
+	currentTime := time.Now().UTC()
+
+	// Check current time
+	blockTime := currentTime
+	assert.True(t, isLatestBlockTimeCurrent(blockTime, 10))
+
+	// Check time from two minutes ago
+	blockTime = currentTime.Add(-time.Minute * 2)
+	assert.True(t, isLatestBlockTimeCurrent(blockTime, 10))
+
+	// Check time from two hours ago
+	blockTime = currentTime.Add(-time.Hour * 2)
+	assert.False(t, isLatestBlockTimeCurrent(blockTime, 10))
+
+	// Check time two minutes in the future
+	blockTime = currentTime.Add(time.Minute * 2)
+	assert.True(t, isLatestBlockTimeCurrent(blockTime, 10))
+}