From 3bff6118d074b0cd86e99cf5329d7757ae1a817b Mon Sep 17 00:00:00 2001 From: rickstaa Date: Fri, 10 Nov 2023 14:52:30 +0100 Subject: [PATCH 1/4] feat(monitor): add reward service error metric This commit adds the `reward_call_errors` prometheus metric when users enable the monitor function. This allows users to see when a reward call failed. --- eth/rewardservice.go | 4 ++++ monitor/census.go | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/eth/rewardservice.go b/eth/rewardservice.go index f7eabaf47..fcb137abf 100644 --- a/eth/rewardservice.go +++ b/eth/rewardservice.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/golang/glog" + "github.com/livepeer/go-livepeer/monitor" ) var ( @@ -57,6 +58,9 @@ func (s *RewardService) Start(ctx context.Context) error { err := s.tryReward() if err != nil { glog.Errorf("Error trying to call reward err=%q", err) + if monitor.Enabled { + monitor.RewardCallError(err.Error()) + } } }() case <-cancelCtx.Done(): diff --git a/monitor/census.go b/monitor/census.go index 171c002b0..f1d49e73e 100644 --- a/monitor/census.go +++ b/monitor/census.go @@ -173,6 +173,9 @@ type ( mMaxGasPrice *stats.Float64Measure mTranscodingPrice *stats.Float64Measure + // Metrics for calling rewards + mRewardCallError *stats.Int64Measure + // Metrics for pixel accounting mMilPixelsProcessed *stats.Float64Measure @@ -318,6 +321,9 @@ func InitCensus(nodeType NodeType, version string) { census.mMaxGasPrice = stats.Float64("max_gas_price", "MaxGasPrice", "gwei") census.mTranscodingPrice = stats.Float64("transcoding_price", "TranscodingPrice", "wei") + // Metrics for calling rewards + census.mRewardCallError = stats.Int64("reward_call_errors", "RewardCallError", "tot") + // Metrics for pixel accounting census.mMilPixelsProcessed = stats.Float64("mil_pixels_processed", "MilPixelsProcessed", "mil pixels") @@ -780,6 +786,15 @@ func InitCensus(nodeType NodeType, version string) { Aggregation: view.LastValue(), }, + // Metrics for calling rewards + { + Name: "reward_call_errors", + Measure: census.mRewardCallError, + Description: "Errors when calling rewards", + TagKeys: baseTags, + Aggregation: view.Sum(), + }, + // Metrics for fast verification { Name: "fast_verification_done", @@ -1683,6 +1698,16 @@ func TranscodingPrice(sender string, price *big.Rat) { } } +// RewardCallError records an error from reward calling +func RewardCallError(sender string) { + if err := stats.RecordWithTags(census.ctx, + []tag.Mutator{tag.Insert(census.kSender, sender)}, + census.mRewardCallError.M(1)); err != nil { + + glog.Errorf("Error recording metrics err=%q", err) + } +} + // Convert wei to gwei func wei2gwei(wei *big.Int) float64 { gwei, _ := new(big.Float).Quo(new(big.Float).SetInt(wei), big.NewFloat(float64(gweiConversionFactor))).Float64() From ad61d754a0268fe0812f46f723b44e72ec6260a5 Mon Sep 17 00:00:00 2001 From: Rick Staa Date: Sat, 11 Nov 2023 13:59:51 +0100 Subject: [PATCH 2/4] refactor: add CHANGELOG_PENDING entry --- CHANGELOG_PENDING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 52cf9f9db..04574e97b 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -18,6 +18,7 @@ - #2911 Set default price with livepeer_cli option 20 (@eliteprox) - #2928 Added `startupAvailabilityCheck` param to skip the availability check on startup (@stronk-dev) +- \#xxx Add `reward_call_errors` Prometheus metric (@rickstaa) #### Transcoder From 46a33d1fae746547a39d344aad654f467f58c544 Mon Sep 17 00:00:00 2001 From: rickstaa Date: Tue, 5 Dec 2023 17:05:50 +0100 Subject: [PATCH 3/4] chore: improve 'CHANGELOG_PENDING' entry --- CHANGELOG_PENDING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 04574e97b..7b33a2322 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -18,7 +18,7 @@ - #2911 Set default price with livepeer_cli option 20 (@eliteprox) - #2928 Added `startupAvailabilityCheck` param to skip the availability check on startup (@stronk-dev) -- \#xxx Add `reward_call_errors` Prometheus metric (@rickstaa) +- #2905 Add `reward_call_errors` Prometheus metric (@rickstaa) #### Transcoder From 8acadbc161363913ed39df9a4bdd79ebb9470a57 Mon Sep 17 00:00:00 2001 From: rickstaa Date: Wed, 20 Dec 2023 13:53:26 +0100 Subject: [PATCH 4/4] feat: add round in reward call error log statement This commit improves the error that is thrown when the reward call fails by including the round number. --- eth/rewardservice.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/rewardservice.go b/eth/rewardservice.go index fcb137abf..bd10a062a 100644 --- a/eth/rewardservice.go +++ b/eth/rewardservice.go @@ -57,7 +57,7 @@ func (s *RewardService) Start(ctx context.Context) error { go func() { err := s.tryReward() if err != nil { - glog.Errorf("Error trying to call reward err=%q", err) + glog.Errorf("Error trying to call reward for round %v err=%q", s.tw.LastInitializedRound(), err) if monitor.Enabled { monitor.RewardCallError(err.Error()) }