feat(monitor): add reward service error metric

CHANGELOG_PENDING.md

@@ -18,6 +18,7 @@
 
 - #2911 Set default price with livepeer_cli option 20 (@eliteprox)
 - #2928 Added `startupAvailabilityCheck` param to skip the availability check on startup (@stronk-dev)
+- #2905 Add `reward_call_errors` Prometheus metric (@rickstaa)
 
 #### Transcoder
 

eth/rewardservice.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/golang/glog"
+	"github.com/livepeer/go-livepeer/monitor"
 )
 
 var (
@@ -56,7 +57,10 @@ func (s *RewardService) Start(ctx context.Context) error {
 			go func() {
 				err := s.tryReward()
 				if err != nil {
-					glog.Errorf("Error trying to call reward err=%q", err)
+					glog.Errorf("Error trying to call reward for round %v err=%q", s.tw.LastInitializedRound(), err)
+					if monitor.Enabled {
+						monitor.RewardCallError(err.Error())
+					}
 				}
 			}()
 		case <-cancelCtx.Done():

monitor/census.go

@@ -173,6 +173,9 @@
 		mMaxGasPrice           *stats.Float64Measure
 		mTranscodingPrice      *stats.Float64Measure
 
+		// Metrics for calling rewards
+		mRewardCallError *stats.Int64Measure
+
 		// Metrics for pixel accounting
 		mMilPixelsProcessed *stats.Float64Measure
 
@@ -318,6 +321,9 @@
 	census.mMaxGasPrice = stats.Float64("max_gas_price", "MaxGasPrice", "gwei")
 	census.mTranscodingPrice = stats.Float64("transcoding_price", "TranscodingPrice", "wei")
 
+	// Metrics for calling rewards
+	census.mRewardCallError = stats.Int64("reward_call_errors", "RewardCallError", "tot")
+
 	// Metrics for pixel accounting
 	census.mMilPixelsProcessed = stats.Float64("mil_pixels_processed", "MilPixelsProcessed", "mil pixels")
 
@@ -780,6 +786,15 @@
 			Aggregation: view.LastValue(),
 		},
 
+		// Metrics for calling rewards
+		{
+			Name:        "reward_call_errors",
+			Measure:     census.mRewardCallError,
+			Description: "Errors when calling rewards",
+			TagKeys:     baseTags,
+			Aggregation: view.Sum(),
+		},
+
 		// Metrics for fast verification
 		{
 			Name:        "fast_verification_done",
@@ -1161,7 +1176,7 @@
 	}
 }
 
 func (cen *censusMetricsCounter) segmentEmerged(nonce, seqNo uint64, profilesNum int) {
 	cen.lock.Lock()
 	defer cen.lock.Unlock()
 	if _, has := cen.emergeTimes[nonce]; !has {
@@ -1178,7 +1193,7 @@
 	census.segmentSourceAppeared(ctx, nonce, seqNo, profile, recordingEnabled)
 }
 
 func (cen *censusMetricsCounter) segmentSourceAppeared(ctx context.Context, nonce, seqNo uint64, profile string, recordingEnabled bool) {
 	segType := segTypeRegular
 	if recordingEnabled {
 		segType = segTypeRec
@@ -1683,6 +1698,16 @@
 	}
 }
 
+// RewardCallError records an error from reward calling
+func RewardCallError(sender string) {
+	if err := stats.RecordWithTags(census.ctx,
+		[]tag.Mutator{tag.Insert(census.kSender, sender)},
+		census.mRewardCallError.M(1)); err != nil {
+
+		glog.Errorf("Error recording metrics err=%q", err)
+	}
+}
+
 // Convert wei to gwei
 func wei2gwei(wei *big.Int) float64 {
 	gwei, _ := new(big.Float).Quo(new(big.Float).SetInt(wei), big.NewFloat(float64(gweiConversionFactor))).Float64()