Merge pull request #221 from lidofinance/fix/val-1175-metrics

Metrics

src/common/prometheus/prometheus.constants.ts

@@ -3,6 +3,7 @@ export const METRICS_PREFIX = 'council_daemon_';
 
 export const METRIC_SENT_MESSAGES = `${METRICS_PREFIX}sent_messages_total`;
 export const METRIC_PAUSE_ATTEMPTS = `${METRICS_PREFIX}pause_deposits_attempts_total`;
+export const METRIC_UNVET_ATTEMPTS = `${METRICS_PREFIX}unvet_attempts_total`;
 
 export const METRIC_RPC_REQUEST_DURATION = `${METRICS_PREFIX}rpc_requests_duration_seconds`;
 export const METRIC_RPC_REQUEST_ERRORS = `${METRICS_PREFIX}rpc_requests_errors`;
@@ -22,6 +23,6 @@ export const METRIC_OPERATORS_KEYS_TOTAL = `${METRICS_PREFIX}operators_keys_tota
 
 export const METRIC_KEYS_API_REQUEST_DURATION = `${METRICS_PREFIX}keys_api_requests_duration_seconds`;
 
-export const METRIC_DUPLICATED_VETTED_UNUSED_KEYS_TOTAL = `${METRICS_PREFIX}duplicated_vetted_unused_keys`;
+export const METRIC_DUPLICATED_KEYS_TOTAL = `${METRICS_PREFIX}duplicated_keys_total`;
 
-export const METRIC_INVALID_KEYS_TOTAL = `${METRICS_PREFIX}invalid_keys`;
+export const METRIC_INVALID_KEYS_TOTAL = `${METRICS_PREFIX}invalid_keys_total`;

src/common/prometheus/prometheus.module.ts

@@ -13,8 +13,9 @@ import {
   PrometheusDepositedKeysProvider,
   PrometheusOperatorsKeysProvider,
   PrometheusKeysApiRequestsProvider,
-  PrometheusVettedUnusedKeysProvider,
+  PrometheusDuplicatedKeysProvider,
   PrometheusInvalidKeysProvider,
+  PrometheusUnvetKeysCounterProvider,
 } from './prometheus.provider';
 import { METRICS_PREFIX, METRICS_URL } from './prometheus.constants';
 
@@ -40,8 +41,9 @@ const providers = [
   PrometheusDepositedKeysProvider,
   PrometheusOperatorsKeysProvider,
   PrometheusKeysApiRequestsProvider,
-  PrometheusVettedUnusedKeysProvider,
+  PrometheusDuplicatedKeysProvider,
   PrometheusInvalidKeysProvider,
+  PrometheusUnvetKeysCounterProvider,
 ];
 
 PrometheusModule.global = true;

src/common/prometheus/prometheus.provider.ts

@@ -17,8 +17,9 @@ import {
   METRIC_DEPOSITED_KEYS_TOTAL,
   METRIC_OPERATORS_KEYS_TOTAL,
   METRIC_KEYS_API_REQUEST_DURATION,
-  METRIC_DUPLICATED_VETTED_UNUSED_KEYS_TOTAL,
+  METRIC_DUPLICATED_KEYS_TOTAL,
   METRIC_INVALID_KEYS_TOTAL,
+  METRIC_UNVET_ATTEMPTS,
 } from './prometheus.constants';
 
 export const PrometheusTransportMessageCounterProvider = makeCounterProvider({
@@ -32,6 +33,11 @@ export const PrometheusPauseDepositsCounterProvider = makeCounterProvider({
   help: 'Attempts to pause deposits',
 });
 
+export const PrometheusUnvetKeysCounterProvider = makeCounterProvider({
+  name: METRIC_UNVET_ATTEMPTS,
+  help: 'Attempts to unvet keys',
+});
+
 export const PrometheusRPCRequestsHistogramProvider = makeHistogramProvider({
   name: METRIC_RPC_REQUEST_DURATION,
   help: 'RPC request duration',
@@ -96,10 +102,10 @@ export const PrometheusKeysApiRequestsProvider = makeHistogramProvider({
   labelNames: ['result', 'status'] as const,
 });
 
-export const PrometheusVettedUnusedKeysProvider = makeGaugeProvider({
-  name: METRIC_DUPLICATED_VETTED_UNUSED_KEYS_TOTAL,
-  help: 'Number of duplicated vetted unused keys events',
-  labelNames: ['stakingModuleId'] as const,
+export const PrometheusDuplicatedKeysProvider = makeGaugeProvider({
+  name: METRIC_DUPLICATED_KEYS_TOTAL,
+  help: 'Number of duplicated keys',
+  labelNames: ['type', 'stakingModuleId'] as const,
 });
 
 export const PrometheusInvalidKeysProvider = makeGaugeProvider({

src/contracts/security/security.service.ts

@@ -2,7 +2,10 @@ import { Signature } from '@ethersproject/bytes';
 import { ContractReceipt } from '@ethersproject/contracts';
 import { Inject, Injectable, LoggerService } from '@nestjs/common';
 import { InjectMetric } from '@willsoto/nestjs-prometheus';
-import { METRIC_PAUSE_ATTEMPTS } from 'common/prometheus';
+import {
+  METRIC_PAUSE_ATTEMPTS,
+  METRIC_UNVET_ATTEMPTS,
+} from 'common/prometheus';
 import { OneAtTime, StakingModuleId } from 'common/decorators';
 import { SecurityAbi, SecurityPauseV2Abi__factory } from 'generated';
 import { RepositoryService } from 'contracts/repository';
@@ -15,6 +18,7 @@ import { WalletService } from 'wallet';
 export class SecurityService {
   constructor(
     @InjectMetric(METRIC_PAUSE_ATTEMPTS) private pauseAttempts: Counter<string>,
+    @InjectMetric(METRIC_UNVET_ATTEMPTS) private unvetAttempts: Counter<string>,
     @Inject(WINSTON_MODULE_NEST_PROVIDER) private logger: LoggerService,
     private providerService: ProviderService,
     private repositoryService: RepositoryService,
@@ -151,9 +155,11 @@ export class SecurityService {
     });
     this.logger.warn('Waiting for block confirmation', { blockNumber });
 
-    await tx.wait();
+    const receipt = await tx.wait();
 
     this.logger.warn('Block confirmation received', { blockNumber });
+
+    return receipt;
   }
 
   /**
@@ -205,16 +211,27 @@ export class SecurityService {
       stakingModuleId,
     });
 
-    await tx.wait();
+    const receipt = await tx.wait();
 
     this.logger.warn('Block confirmation received', {
       blockNumber,
       stakingModuleId,
     });
+
+    return receipt;
   }
 
   /**
-   * Signs a message to unvet keys
+   * Signs a message to unvet keys for a staking module.
+   *
+   * @param nonce - The nonce for the staking module.
+   * @param blockNumber - The block number at which the message is signed.
+   * @param blockHash - The hash of the block corresponding to the block number.
+   * @param stakingModuleId - The ID of the target staking module.
+   * @param operatorIds - A string containing the IDs of the operators whose keys are being unvetted.
+   * @param vettedKeysByOperator - A string representing the new staking limit amount per operator.
+   *
+   * @returns A signature object containing the signed data.
    */
   public async signUnvetData(
     nonce: number,
@@ -238,7 +255,17 @@ export class SecurityService {
   }
 
   /**
-   * Send transaction to unvet signing keys
+   * Sends a transaction to unvet signing keys for a staking module.
+   *
+   * @param nonce - The nonce for the staking module.
+   * @param blockNumber - The block number at which the message is signed.
+   * @param blockHash - The hash of the block corresponding to the block number.
+   * @param stakingModuleId - The ID of the target staking module.
+   * @param operatorIds - A string containing the IDs of the operators whose keys are being unvetted.
+   * @param vettedKeysByOperator - A string representing the new staking limit amount per operator.
+   * @param signature - The signature of the message, containing `r` and `_vs`.
+   *
+   * @returns The transaction receipt or `void` if the transaction fails.
    */
   @OneAtTime()
   public async unvetSigningKeys(
@@ -254,6 +281,7 @@ export class SecurityService {
       stakingModuleId,
       blockNumber,
     });
+    this.unvetAttempts.inc();
 
     const contract = this.getContractWithSigner();
 
@@ -281,12 +309,14 @@ export class SecurityService {
       stakingModuleId,
     });
 
-    await tx.wait();
+    const receipt = await tx.wait();
 
     this.logger.warn('Block confirmation received', {
       blockNumber,
       stakingModuleId,
     });
+
+    return receipt;
   }
 
   /**

src/guardian/guardian-metrics/guardian-metrics.service.ts

@@ -8,7 +8,7 @@ import {
   METRIC_OPERATORS_KEYS_TOTAL,
   METRIC_INTERSECTIONS_TOTAL,
   METRIC_INVALID_KEYS_TOTAL,
-  METRIC_DUPLICATED_VETTED_UNUSED_KEYS_TOTAL,
+  METRIC_DUPLICATED_KEYS_TOTAL,
 } from 'common/prometheus';
 import { Gauge } from 'prom-client';
 
@@ -27,8 +27,8 @@ export class GuardianMetricsService {
     @InjectMetric(METRIC_INTERSECTIONS_TOTAL)
     private intersectionsCounter: Gauge<string>,
 
-    @InjectMetric(METRIC_DUPLICATED_VETTED_UNUSED_KEYS_TOTAL)
-    private duplicatedVettedUnusedKeysCounter: Gauge<string>,
+    @InjectMetric(METRIC_DUPLICATED_KEYS_TOTAL)
+    private duplicatedKeysCounter: Gauge<string>,
 
     @InjectMetric(METRIC_INVALID_KEYS_TOTAL)
     private invalidKeysCounter: Gauge<string>,
@@ -129,13 +129,29 @@ export class GuardianMetricsService {
   /**
    * increment duplicated vetted unused keys event counter
    */
-  public collectDuplicatedVettedUnusedKeysMetrics(
+  public collectDuplicatedKeysMetrics(
     stakingModuleId: number,
-    duplicatedVettedUnusedKeysCount: number,
+    allUnresolved: number,
+    unresolved: number,
+    allVettedUnused: number,
+    vettedUnused: number,
   ) {
-    this.duplicatedVettedUnusedKeysCounter.set(
-      { stakingModuleId },
-      duplicatedVettedUnusedKeysCount,
+    this.duplicatedKeysCounter.set(
+      { stakingModuleId, type: 'all_unresolved' },
+      allUnresolved,
+    );
+    this.duplicatedKeysCounter.set(
+      { stakingModuleId, type: 'vetted_unused_unresolved' },
+      unresolved,
+    );
+    // resolved - SigningKeyAdded event exists
+    this.duplicatedKeysCounter.set(
+      { stakingModuleId, type: 'all_vetted_unused' },
+      allVettedUnused,
+    );
+    this.duplicatedKeysCounter.set(
+      { stakingModuleId, type: 'vetted_unused' },
+      vettedUnused,
     );
   }
 

src/staking-router/staking-router.module.ts

@@ -4,13 +4,15 @@ import { StakingRouterService } from './staking-router.service';
 import { SecurityModule } from 'contracts/security';
 import { StakingModuleGuardModule } from 'guardian/staking-module-guard';
 import { KeysDuplicationCheckerModule } from 'guardian/duplicates';
+import { GuardianMetricsModule } from 'guardian/guardian-metrics';
 
 @Module({
   imports: [
     ConfigModule,
     SecurityModule,
     StakingModuleGuardModule,
     KeysDuplicationCheckerModule,
+    GuardianMetricsModule,
   ],
   providers: [StakingRouterService],
   exports: [StakingRouterService],

src/staking-router/staking-router.service.ts

@@ -8,6 +8,7 @@ import { SROperatorListWithModule } from 'keys-api/interfaces/SROperatorListWith
 import { SecurityService } from 'contracts/security';
 import { StakingModuleGuardService } from 'guardian/staking-module-guard';
 import { KeysDuplicationCheckerService } from 'guardian/duplicates';
+import { GuardianMetricsService } from 'guardian/guardian-metrics';
 
 type State = {
   operatorsByModules: SROperatorListWithModule[];
@@ -23,6 +24,7 @@ export class StakingRouterService {
     private securityService: SecurityService,
     private stakingModuleGuardService: StakingModuleGuardService,
     private keysDuplicationCheckerService: KeysDuplicationCheckerService,
+    private guardianMetricsService: GuardianMetricsService,
   ) {}
 
   /**
@@ -96,17 +98,37 @@ export class StakingRouterService {
             stakingModuleData,
             blockData,
           );
-        stakingModuleData.duplicatedKeys = this.filterModuleNotVettedUnusedKeys(
+        const allDuplicatedKeys = this.getModuleKeys(
           stakingModuleData.stakingModuleAddress,
-          stakingModuleData.vettedUnusedKeys,
           duplicates,
         );
-        stakingModuleData.unresolvedDuplicatedKeys =
-          this.filterModuleNotVettedUnusedKeys(
-            stakingModuleData.stakingModuleAddress,
-            stakingModuleData.vettedUnusedKeys,
-            unresolved,
-          );
+        stakingModuleData.duplicatedKeys = this.getVettedUnusedKeys(
+          stakingModuleData.vettedUnusedKeys,
+          allDuplicatedKeys,
+        );
+
+        const allUnresolved = this.getModuleKeys(
+          stakingModuleData.stakingModuleAddress,
+          unresolved,
+        );
+
+        stakingModuleData.unresolvedDuplicatedKeys = this.getVettedUnusedKeys(
+          stakingModuleData.vettedUnusedKeys,
+          allUnresolved,
+        );
+
+        this.guardianMetricsService.collectDuplicatedKeysMetrics(
+          stakingModuleData.stakingModuleId,
+          allUnresolved.length,
+          stakingModuleData.unresolvedDuplicatedKeys.length,
+          allDuplicatedKeys.length,
+          stakingModuleData.duplicatedKeys.length,
+        );
+
+        this.guardianMetricsService.collectInvalidKeysMetrics(
+          stakingModuleData.stakingModuleId,
+          stakingModuleData.invalidKeys.length,
+        );
 
         this.logger.log('Keys check state', {
           stakingModuleId: stakingModuleData.stakingModuleId,
@@ -121,23 +143,24 @@ export class StakingRouterService {
     );
   }
 
+  private getModuleKeys(stakingModuleAddress: string, keys: RegistryKey[]) {
+    return keys.filter((key) => key.moduleAddress === stakingModuleAddress);
+  }
+
   /**
    * filter from the list all keys that are not vetted as unused
    */
-  public filterModuleNotVettedUnusedKeys(
-    stakingModuleAddress: string,
+  public getVettedUnusedKeys(
     vettedUnusedKeys: RegistryKey[],
     keys: RegistryKey[],
   ) {
-    const vettedUnused = keys
-      .filter((key) => key.moduleAddress === stakingModuleAddress)
-      .filter((key) => {
-        const r = vettedUnusedKeys.some(
-          (k) => k.index == key.index && k.operatorIndex == key.operatorIndex,
-        );
+    const vettedUnused = keys.filter((key) => {
+      const r = vettedUnusedKeys.some(
+        (k) => k.index == key.index && k.operatorIndex == key.operatorIndex,
+      );
 
-        return r;
-      });
+      return r;
+    });
 
     return vettedUnused;
   }

test/duplicates-v3.e2e-spec.ts

@@ -756,9 +756,9 @@ describe('Deposits in case of duplicates', () => {
       'handleCorrectKeys',
     );
 
-    const filterModuleNotVettedUnusedKeys = jest.spyOn(
+    const getVettedUnusedKeys = jest.spyOn(
       stakingRouterService,
-      'filterModuleNotVettedUnusedKeys',
+      'getVettedUnusedKeys',
     );
     await guardianService.handleNewBlock();
     await new Promise((res) => setTimeout(res, SLEEP_FOR_RESULT));
@@ -780,30 +780,20 @@ describe('Deposits in case of duplicates', () => {
         stakingModuleId: sdvtModule.id,
       }),
     );
-    expect(filterModuleNotVettedUnusedKeys).toBeCalledTimes(4);
-    expect(filterModuleNotVettedUnusedKeys).toHaveBeenCalledWith(
-      NOP_REGISTRY,
+    expect(getVettedUnusedKeys).toBeCalledTimes(4);
+    expect(getVettedUnusedKeys).toHaveBeenCalledWith(
       expect.arrayContaining([keys[0]]),
       expect.arrayContaining([keys[1]]),
     );
 
     //unresolved duplicates
-    expect(filterModuleNotVettedUnusedKeys).toHaveBeenCalledWith(
-      NOP_REGISTRY,
+    expect(getVettedUnusedKeys).toHaveBeenCalledWith(
       expect.arrayContaining([keys[0]]),
       [],
     );
-    expect(filterModuleNotVettedUnusedKeys).toHaveBeenCalledWith(
-      SIMPLE_DVT,
-      [],
-      [],
-    );
+    expect(getVettedUnusedKeys).toHaveBeenCalledWith([], []);
     //unresolved duplicates
-    expect(filterModuleNotVettedUnusedKeys).toHaveBeenCalledWith(
-      SIMPLE_DVT,
-      [],
-      [],
-    );
+    expect(getVettedUnusedKeys).toHaveBeenCalledWith([], []);
 
     expect(handleCorrectKeys).toBeCalledTimes(2);
     expect(handleCorrectKeys).toHaveBeenCalledWith(