diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 52cf9f9dbe..7b33a2322f 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -18,6 +18,7 @@ - #2911 Set default price with livepeer_cli option 20 (@eliteprox) - #2928 Added `startupAvailabilityCheck` param to skip the availability check on startup (@stronk-dev) +- #2905 Add `reward_call_errors` Prometheus metric (@rickstaa) #### Transcoder diff --git a/eth/rewardservice.go b/eth/rewardservice.go index f7eabaf470..bd10a062a3 100644 --- a/eth/rewardservice.go +++ b/eth/rewardservice.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/golang/glog" + "github.com/livepeer/go-livepeer/monitor" ) var ( @@ -56,7 +57,10 @@ func (s *RewardService) Start(ctx context.Context) error { go func() { err := s.tryReward() if err != nil { - glog.Errorf("Error trying to call reward err=%q", err) + glog.Errorf("Error trying to call reward for round %v err=%q", s.tw.LastInitializedRound(), err) + if monitor.Enabled { + monitor.RewardCallError(err.Error()) + } } }() case <-cancelCtx.Done(): diff --git a/monitor/census.go b/monitor/census.go index 171c002b05..f1d49e73e7 100644 --- a/monitor/census.go +++ b/monitor/census.go @@ -173,6 +173,9 @@ type ( mMaxGasPrice *stats.Float64Measure mTranscodingPrice *stats.Float64Measure + // Metrics for calling rewards + mRewardCallError *stats.Int64Measure + // Metrics for pixel accounting mMilPixelsProcessed *stats.Float64Measure @@ -318,6 +321,9 @@ func InitCensus(nodeType NodeType, version string) { census.mMaxGasPrice = stats.Float64("max_gas_price", "MaxGasPrice", "gwei") census.mTranscodingPrice = stats.Float64("transcoding_price", "TranscodingPrice", "wei") + // Metrics for calling rewards + census.mRewardCallError = stats.Int64("reward_call_errors", "RewardCallError", "tot") + // Metrics for pixel accounting census.mMilPixelsProcessed = stats.Float64("mil_pixels_processed", "MilPixelsProcessed", "mil pixels") @@ -780,6 +786,15 @@ func InitCensus(nodeType NodeType, version string) { Aggregation: view.LastValue(), }, + // Metrics for calling rewards + { + Name: "reward_call_errors", + Measure: census.mRewardCallError, + Description: "Errors when calling rewards", + TagKeys: baseTags, + Aggregation: view.Sum(), + }, + // Metrics for fast verification { Name: "fast_verification_done", @@ -1683,6 +1698,16 @@ func TranscodingPrice(sender string, price *big.Rat) { } } +// RewardCallError records an error from reward calling +func RewardCallError(sender string) { + if err := stats.RecordWithTags(census.ctx, + []tag.Mutator{tag.Insert(census.kSender, sender)}, + census.mRewardCallError.M(1)); err != nil { + + glog.Errorf("Error recording metrics err=%q", err) + } +} + // Convert wei to gwei func wei2gwei(wei *big.Int) float64 { gwei, _ := new(big.Float).Quo(new(big.Float).SetInt(wei), big.NewFloat(float64(gweiConversionFactor))).Float64()