add telemetry and logs for liquidation daemon (backport #2122) (#2128)

Co-authored-by: Jay Yu <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

protocol/daemons/liquidation/client/grpc_helper.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"time"
 
+	errorsmod "cosmossdk.io/errors"
 	"github.com/cosmos/cosmos-sdk/telemetry"
 	"github.com/cosmos/cosmos-sdk/types/grpc"
 	"github.com/cosmos/cosmos-sdk/types/query"
@@ -271,9 +272,20 @@ func (c *Client) SendLiquidatableSubaccountIds(
 		)...,
 	)
 
+	telemetry.ModuleSetGauge(
+		metrics.LiquidationDaemon,
+		float32(len(requests)),
+		metrics.NumRequests,
+		metrics.Count,
+	)
+
 	for _, req := range requests {
 		if _, err := c.LiquidationServiceClient.LiquidateSubaccounts(ctx, req); err != nil {
-			return err
+			return errorsmod.Wrapf(
+				err,
+				"failed to send liquidatable subaccount ids to protocol at block height %d",
+				blockHeight,
+			)
 		}
 	}
 

protocol/daemons/liquidation/client/grpc_helper_test.go

@@ -590,7 +590,11 @@ func TestSendLiquidatableSubaccountIds(t *testing.T) {
 				tc.subaccountOpenPositionInfo,
 				1000,
 			)
-			require.Equal(t, tc.expectedError, err)
+			if tc.expectedError != nil {
+				require.ErrorContains(t, err, tc.expectedError.Error())
+			} else {
+				require.NoError(t, err)
+			}
 		})
 	}
 }

protocol/daemons/liquidation/client/sub_task_runner.go

@@ -26,7 +26,9 @@ type SubTaskRunner interface {
 	) error
 }
 
-type SubTaskRunnerImpl struct{}
+type SubTaskRunnerImpl struct {
+	lastLoopBlockHeight uint32
+}
 
 // Ensure SubTaskRunnerImpl implements the SubTaskRunner interface.
 var _ SubTaskRunner = (*SubTaskRunnerImpl)(nil)
@@ -50,6 +52,19 @@ func (s *SubTaskRunnerImpl) RunLiquidationDaemonTaskLoop(
 		return err
 	}
 
+	// Skip the loop if no new block has been committed.
+	// Note that lastLoopBlockHeight is initialized to 0, so the first loop will always run.
+	if lastCommittedBlockHeight == s.lastLoopBlockHeight {
+		daemonClient.logger.Info(
+			"Skipping liquidation daemon task loop as no new block has been committed",
+			"blockHeight", lastCommittedBlockHeight,
+		)
+		return nil
+	}
+
+	// Update the last loop block height.
+	s.lastLoopBlockHeight = lastCommittedBlockHeight
+
 	// 1. Fetch all information needed to calculate total net collateral and margin requirements.
 	subaccounts,
 		perpInfos,
@@ -120,13 +135,21 @@ func (c *Client) FetchApplicationStateAtBlockHeight(
 	// Subaccounts
 	subaccounts, err = c.GetAllSubaccounts(queryCtx, liqFlags.QueryPageLimit)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errorsmod.Wrapf(
+			err,
+			"failed to fetch subaccounts at block height %d",
+			blockHeight,
+		)
 	}
 
 	// Market prices
 	marketPrices, err := c.GetAllMarketPrices(queryCtx, liqFlags.QueryPageLimit)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errorsmod.Wrapf(
+			err,
+			"failed to fetch market prices at block height %d",
+			blockHeight,
+		)
 	}
 	marketPricesMap := lib.UniqueSliceToMap(marketPrices, func(m pricestypes.MarketPrice) uint32 {
 		return m.Id
@@ -135,13 +158,21 @@ func (c *Client) FetchApplicationStateAtBlockHeight(
 	// Perpetuals
 	perpetuals, err := c.GetAllPerpetuals(queryCtx, liqFlags.QueryPageLimit)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errorsmod.Wrapf(
+			err,
+			"failed to fetch perpetuals at block height %d",
+			blockHeight,
+		)
 	}
 
 	// Liquidity tiers
 	liquidityTiers, err := c.GetAllLiquidityTiers(queryCtx, liqFlags.QueryPageLimit)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errorsmod.Wrapf(
+			err,
+			"failed to fetch liquidity tiers at block height %d",
+			blockHeight,
+		)
 	}
 	liquidityTiersMap := lib.UniqueSliceToMap(liquidityTiers, func(l perptypes.LiquidityTier) uint32 {
 		return l.Id

protocol/lib/metrics/constants.go

@@ -314,6 +314,7 @@ const (
 	GetSubaccountsFromKey                    = "get_subaccounts_from_key"
 	LiquidatableSubaccountIds                = "liquidatable_subaccount_ids"
 	LiquidationDaemon                        = "liquidation_daemon"
+	NumRequests                              = "num_requests"
 	NegativeTncSubaccountIds                 = "negative_tnc_subaccount_ids"
 	PageLimit                                = "page_limit"
 	SendLiquidatableSubaccountIds            = "send_liquidatable_subaccount_ids"